diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index 7d88165009b..eee20ba3f81 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -459,6 +459,20 @@ Environment tests use the selected account and remove their temporary homes after probing. A disconnected target can prevent remote cleanup; the run reports the resulting failure rather than silently selecting another account. +Codex CLI runs in an in-place SSH workspace keep conversation files and native +SQLite indexes in a private remote home scoped to the company, agent, AI account, +and conversation. The home path stays stable across turns and token refreshes. +Each turn installs the current managed files, copies refreshed authentication +back, and removes everything except conversation state, including on provider +failure. Credentials and managed MCP configuration do not remain there between +turns. Switching accounts selects a different home and starts a new conversation. +The cache lives below the environment's remote workspace root. Removing that +root also removes the retained conversation state. +Older SSH host-auth sessions keep their existing remote home. A managed SSH +session saved before conversation retention has no retained state reference and +starts a fresh conversation instead of reusing an unavailable thread. This reset +is recorded as `codex_ssh_session_state_unavailable`; existing saved files remain. + ACP SSH sessions use the existing remote process bridge and session reuse lifecycle. A compatible session can retain its private remote home; workspace sync and credential refresh run at turn teardown, and the session disposal hook diff --git a/packages/adapters/codex-local/src/server/execute.remote.test.ts b/packages/adapters/codex-local/src/server/execute.remote.test.ts index fd36719149c..4329b22afc5 100644 --- a/packages/adapters/codex-local/src/server/execute.remote.test.ts +++ b/packages/adapters/codex-local/src/server/execute.remote.test.ts @@ -1,4 +1,6 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -27,7 +29,7 @@ const { prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })), restoreWorkspaceFromSshExecution: vi.fn(async () => undefined), runSshCommand: vi.fn(async (_spec?: unknown, _command?: string) => ({ stdout: Buffer.from("{}").toString("base64"), stderr: "" })), - syncDirectoryToSsh: vi.fn(async () => undefined), + syncDirectoryToSsh: vi.fn(async (_input?: { localDir: string; remoteDir: string }) => undefined), startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({ env: { PAPERCLIP_API_URL: "http://127.0.0.1:4310", @@ -88,6 +90,109 @@ describe("codex remote execution", () => { } }); + it.each([false, true])("retains a managed SSH Codex conversation across turns and removes credentials (second turn throws=%s)", async fails => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-ssh-resume-")); + cleanupDirs.push(root); + const workspace = path.join(root, "workspace"), home = path.join(root, "account"), secondHome = path.join(root, "next-invocation"); + await mkdir(workspace); await mkdir(home); await mkdir(secondHome); + const auth = (account: string, marker: string, last_refresh: string) => JSON.stringify({ tokens: { + account_id: account, access_token: `fixture-access-${marker}`, refresh_token: `fixture-refresh-${marker}`, id_token: `fixture-id-${marker}`, + }, last_refresh }); + const credential = auth("selected", "before", "2000-01-01T00:00:00Z"); + const refreshed = auth("selected", "after", new Date().toISOString()); + const otherCredential = auth("other", "other", "2000-01-01T00:00:00Z"); + await writeFile(path.join(home, "auth.json"), credential); + await writeFile(path.join(secondHome, "auth.json"), credential); + const executeShell = promisify(execFile); + const originalRun = runChildProcess.getMockImplementation()!; + let firstRemoteHome: string | undefined; + let otherRemoteHome: string | undefined; + const privateRoots: string[] = []; + let calls = 0; + runSshCommand.mockImplementation(async (_spec, command) => { + const result = await executeShell("sh", ["-c", command!]); + if (command!.includes("mktemp -d")) privateRoots.push(result.stdout.trim()); + return result; + }); + syncDirectoryToSsh.mockImplementation(async input => { await cp(input!.localDir, input!.remoteDir, { recursive: true }); }); + runChildProcess.mockImplementation(async (...args: unknown[]) => { + const env = (args[3] as { env: Record }).env; + const remoteHome = env.CODEX_HOME!; + calls++; + expect(await readFile(path.join(remoteHome, "auth.json"), "utf8")).toBe(calls === 1 ? credential : calls === 2 ? refreshed : otherCredential); + if (calls === 1) { + firstRemoteHome = remoteHome; + await mkdir(path.join(remoteHome, "sessions")); + await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "fixture conversation"); + await writeFile(path.join(remoteHome, "state_5.sqlite"), "fixture native index"); + await writeFile(path.join(remoteHome, "auth.json"), refreshed); + } else if (calls === 2) { + expect(remoteHome).toBe(firstRemoteHome); + expect(args[2]).toContain("resume"); + expect(args[2]).toContain("ssh-thread"); + expect(await readFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation"); + expect(await readFile(path.join(remoteHome, "state_5.sqlite"), "utf8")).toBe("fixture native index"); + if (fails) throw new Error("fixture provider failed"); + } else { + if (calls === 3) otherRemoteHome = remoteHome; + expect(remoteHome).not.toBe(firstRemoteHome); + expect(args[2]).not.toContain("resume"); + await expect(readFile(path.join(remoteHome, "sessions", "rollout.jsonl"))).rejects.toThrow("ENOENT"); + if (calls === 3) { + await mkdir(path.join(remoteHome, "sessions")); + await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "other conversation"); + } + } + return { exitCode: 0, signal: null, timedOut: false, stdout: JSON.stringify({ type: "thread.started", thread_id: calls < 3 ? "ssh-thread" : "other-thread" })+'\n{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', stderr: "", pid: 123, startedAt: new Date().toISOString() }; + }); + const ctx = { + agent: { id: "agent-1", companyId: "company-1", name: "Selected agent", adapterType: "codex_local", adapterConfig: {} }, + config: { engine: "cli", command: "codex", managedAiConnection: { identity: "generation-one", sessionIdentity: "selected-account", provider: "openai", method: "subscription" }, env: { CODEX_HOME: home, OPENAI_API_KEY: "" } }, + context: { paperclipWorkspace: { cwd: workspace, source: "project_primary" } }, + executionTarget: { kind: "remote" as const, transport: "ssh" as const, remoteCwd: workspace, + spec: { useHostAuth: false, host: "127.0.0.1", port: 2222, username: "fixture", remoteCwd: workspace, remoteWorkspacePath: workspace, privateKey: null, knownHosts: null, strictHostKeyChecking: true }, + workspaceRealization: { mode: "in_place" as const, authoritativeRoot: workspace, pathAliases: [], outboundRestorePaths: [] } }, onLog: async () => {}, + }; + try { + const first = await execute({ ...ctx, runId: "first", runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: "task-one" } }); + expect(first.sessionId).toBe("ssh-thread"); + expect(await readFile(path.join(home, "auth.json"), "utf8")).toBe(refreshed); + await writeFile(path.join(secondHome, "auth.json"), refreshed); + expect(await readdir(firstRemoteHome!)).toEqual(expect.arrayContaining(["sessions", "state_5.sqlite"])); + expect(await readdir(firstRemoteHome!)).not.toContain("auth.json"); + const second = execute({ ...ctx, runId: "second", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "generation-two" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } }); + if (fails) await expect(second).rejects.toThrow("fixture provider failed"); + else expect((await second).sessionId).toBe("ssh-thread"); + expect(calls).toBe(2); + expect((await readdir(firstRemoteHome!)).sort()).toEqual(["sessions", "state_5.sqlite"]); + expect((await stat(firstRemoteHome!)).mode & 0o077).toBe(0); + if (!fails) { + await writeFile(path.join(secondHome, "auth.json"), otherCredential); + const switched = await execute({ ...ctx, runId: "other-account", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } }); + expect(switched.sessionId).toBe("other-thread"); + expect(switched.sessionParams?.sshCodexAccountScope).not.toBe(first.sessionParams?.sshCodexAccountScope); + expect(switched.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } }); + expect(await readdir(otherRemoteHome!)).not.toContain("auth.json"); + expect(await readFile(path.join(firstRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation"); + const oldShape = { sessionId: switched.sessionId!, cwd: workspace, remoteExecution: switched.sessionParams!.remoteExecution }; + const legacy = await execute({ ...ctx, runId: "old-saved-shape", config: { ...ctx.config, + managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } }, + runtime: { sessionId: switched.sessionId!, sessionParams: oldShape, sessionDisplayId: switched.sessionId!, taskKey: "task-one" } }); + expect(legacy.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } }); + expect(await readFile(path.join(otherRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("other conversation"); + } + for (const dir of privateRoots) await expect(stat(dir)).rejects.toThrow("ENOENT"); + } finally { + runChildProcess.mockImplementation(originalRun); + runSshCommand.mockReset().mockResolvedValue({ stdout: Buffer.from("{}").toString("base64"), stderr: "" }); + syncDirectoryToSsh.mockReset().mockResolvedValue(undefined); + } + }); + it("stages only the selected agent subscription and returns SSH refreshes to that account", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-selected-ssh-")); cleanupDirs.push(root); diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index 499ade3b259..8788a399b02 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url"; import { inferOpenAiCompatibleBiller, resolveManagedOpenAiBilling, type AdapterExecutionContext, type AdapterExecutionResult } from "@paperclipai/adapter-utils"; import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js"; import { copyBackCodexAuth } from "./codex-auth-copyback.js"; +import { prepareCodexSshSessionState } from "./ssh-session-state.js"; import { ensureCodexAuthCacheEntryDir, isCodexAuthCacheEnabled, @@ -783,6 +784,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise> | null = null; let observabilityProfile: Awaited> | null = null; let observabilityPreparationFailed = false; try { @@ -894,9 +896,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise - void (await copyBackCodexAuth({ - readSandboxAuth: () => readFile(path.posix.join(assetDir, "auth.json")), + restore: async ({ assetDir, readFile }) => { + try { await copyBackCodexAuth({ + readSandboxAuth: () => readFile(path.posix.join(sshSessionState?.home ?? assetDir, "auth.json")), hostAuthPath: path.join(config.managedAiConnection ? effectiveCodexHome : resolveSharedCodexHomeDir(process.env), "auth.json"), log: (line) => onLog("stdout", `${line}\n`), // Additive cache write (sandbox to host): also cache the @@ -907,7 +909,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise ensureCodexAuthCacheEntryDir(process.env, accountId, agent.companyId), env: process.env, - })), + }); } finally { await sshSessionState?.cleanup(); } + }, // No `exclude` denylist: `stagedCodexHomeDir` already contains // ONLY the allowlisted files (auth/config/skills), so there is // nothing to filter out. @@ -928,10 +931,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise> = null; let executionError: unknown = null; try { - const remoteCodexHome = executionTargetIsRemote && !useSshHostAuth + let remoteCodexHome = executionTargetIsRemote && !useSshHostAuth ? preparedExecutionTargetRuntime?.assetDirs.home ?? path.posix.join(effectiveExecutionCwd, ".paperclip-runtime", "codex", "home") : null; + if (executionTarget?.kind === "remote" && executionTarget.transport === "ssh" && executionTarget.spec.useHostAuth === false + && config.managedAiConnection && targetWorkspaceRealization?.mode === "in_place" && remoteCodexHome) { + const account = parseObject(config.managedAiConnection); + const identity = asString(account.sessionIdentity, asString(account.identity, "")); + if (!identity) throw new Error("Managed SSH Codex execution requires an account identity"); + sshSessionState = await prepareCodexSshSessionState({ + remoteRoot: executionTarget.spec.remoteWorkspacePath, companyId: agent.companyId, agentId: agent.id, + accountIdentity: identity, executionIdentity: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget), + runId, saved: parseObject(runtime.sessionParams), stagedHome: remoteCodexHome, + run: async command => { + const result = await runAdapterExecutionTargetShellCommand(runId, runtimeExecutionTarget, command, { cwd: effectiveExecutionCwd, env: {}, timeoutSec: 15 }); + if (result.exitCode !== 0 || result.timedOut) throw new Error("Cannot prepare or clean up SSH Codex conversation state"); + }, + }); + remoteCodexHome = sshSessionState.home; + } await emitSandboxAuthPrecedenceWarningIfNeeded({ runId, target: runtimeExecutionTarget, @@ -1138,6 +1157,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise 0 && + (!sshSessionState || sshSessionState.canResume) && (runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(effectiveExecutionCwd)) && adapterExecutionTargetSessionMatches(runtimeRemoteExecution, runtimeExecutionTarget); const codexTransientFallbackMode = readCodexTransientFallbackMode(context); @@ -1525,7 +1545,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise; + stagedHome: string; + run: (command: string) => Promise; +}) { + const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); + const scope = hash([input.companyId, input.agentId, input.accountIdentity, input.executionIdentity]); + const savedId = input.saved.sshCodexConversationId; + const canResume = input.saved.sshCodexAccountScope === scope && typeof savedId === "string" && /^[a-f0-9]{64}$/.test(savedId); + const conversationId = canResume ? savedId : hash(input.runId); + const home = path.posix.join(input.remoteRoot, ".paperclip-runtime", "codex", "conversations", scope, conversationId); + const prefix = `set -eu\numask 077\nhome=${shellQuote(home)}\n[ ! -L "$home" ]\n`; + const cleanup = () => input.run(`${prefix}if [ -d "$home" ]; then\n${STATE_ONLY}\nfi`); + try { + await input.run(`${prefix}mkdir -p "$home"\nchmod 700 "$home"\n${STATE_ONLY}\ncp -RP ${shellQuote(input.stagedHome + "/.")} "$home/"`); + } catch (error) { + await cleanup(); + throw error; + } + return { home, scope, conversationId, canResume, cleanup }; +} diff --git a/packages/adapters/grok-local/src/server/execute.test.ts b/packages/adapters/grok-local/src/server/execute.test.ts index ba18635e386..b741f0640c1 100644 --- a/packages/adapters/grok-local/src/server/execute.test.ts +++ b/packages/adapters/grok-local/src/server/execute.test.ts @@ -64,7 +64,10 @@ vi.mock("@paperclipai/adapter-utils/execution-target", () => ({ prepareAdapterExecutionTargetRuntime: (...args: unknown[]) => (mocks.prepareRuntimeMock as (...args: unknown[]) => unknown)(...args), readAdapterExecutionTarget: (input: { executionTarget?: unknown }) => input.executionTarget ?? - (mocks.state.isRemote ? { kind: "remote", transport: "ssh" } : { kind: "local" }), + (mocks.state.isRemote ? { kind: "remote", transport: "ssh", remoteCwd: "/remote/workspace", spec: { + host: "fixture.invalid", port: 22, username: "fixture", remoteCwd: "/remote/workspace", remoteWorkspacePath: "/remote/workspace", + privateKey: null, knownHosts: null, strictHostKeyChecking: true, + } } : { kind: "local" }), resolveAdapterExecutionTargetCommandForLogs: (...args: unknown[]) => (mocks.resolveCommandForLogsMock as (...args: unknown[]) => unknown)(...args), resolveAdapterExecutionTargetTimeoutSec: (_target: unknown, timeoutSec: number) => timeoutSec, diff --git a/server/src/__tests__/ai-connection-router.test.ts b/server/src/__tests__/ai-connection-router.test.ts index 329feb58dbc..6f6ca01fc94 100644 --- a/server/src/__tests__/ai-connection-router.test.ts +++ b/server/src/__tests__/ai-connection-router.test.ts @@ -27,6 +27,8 @@ import { legacyExecutionNeedsReconciliation } from "../services/legacy-execution import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { dotRunnerBroker } from "../services/dot-runner-broker.js"; +import { environmentService } from "../services/environments.js"; +import * as executionConfiguration from "../services/heartbeat/execution-configuration.js"; const captureRunFailure = vi.hoisted(() => vi.fn()); vi.mock("../sentry.js", async (importOriginal) => ({ @@ -446,6 +448,52 @@ describe("durable, authorized connection routing", () => { expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: String(selected.runtimeConfig.model), taskSessionParams: { __paperclipConfiguredModel: selected.runtimeConfig.model, __paperclipConfigFingerprint: previous.fingerprint, __paperclipConfigFingerprintVersion: previous.version, __paperclipConfigCategoryFingerprints: previous.categoryFingerprints }, configMetadata: current, compatibleConfigMetadata }).reset).toBe(false); } finally { await before.cleanup(); await after.cleanup(); } }); + it.each([ + { useHostAuth: undefined, nativeHomeMode: undefined }, { useHostAuth: true, nativeHomeMode: undefined }, + { useHostAuth: false, nativeHomeMode: undefined }, { useHostAuth: false, nativeHomeMode: "preconfigured" }, + ])("applies SSH authentication before selecting a busy pool (%j)", async ({ useHostAuth, nativeHomeMode }) => { + const original = await instanceSettingsService(db).getExperimental(); + await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, enableNativeRunner: true, enableEnvironments: true, enableWorktreeRunExecution: true }); + const pool = await makePool({ members: [members[0]!] }); + const environment = await environmentService(db).create(companyId, { name: `SSH routing fixture ${useHostAuth} ${nativeHomeMode}`, driver: "ssh", status: "active", envVars: {}, config: { + host: "ssh.example.test", username: "fixture", remoteWorkspacePath: "/workspace", ...(useHostAuth === undefined ? {} : { useHostAuth }), + } }); + const configured = { provider: "acpx", acpxAgent: "claude", model: "fixture-host-model", ...(nativeHomeMode ? { homeMode: nativeHomeMode } : {}) }; + const [agent] = await db.insert(agents).values({ companyId, name: "SSH routing fixture", status: "idle", adapterType: "paperclip_runner", + adapterConfig: configured, defaultEnvironmentId: environment.id, + runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id }, heartbeat: { enabled: false, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "SSH pool routing fixture", status: "todo", assigneeAgentId: agent.id, responsibleUserId: "alice", createdByUserId: "alice" }).returning(); + proposal.mockClear(); + proposal.mockImplementation(async () => ({ kind: "exhausted", retryAt: new Date(Date.now() + 60_000).toISOString(), skipped: {} })); + const preparation = vi.spyOn(executionConfiguration, "prepareHeartbeatExecutionConfiguration").mockRejectedValue(new Error("fixture stopped before provider configuration")); + const heartbeat = heartbeatService(db, { pluginWorkerManager: worker, runtimeEnv: {} }); + try { + const run = await heartbeat.wakeup(agent.id, { source: "assignment", reason: "issue_assigned", triggerDetail: "system", + payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, responsibleUserId: "alice" } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const saved = await heartbeat.getRun(run!.id); + if (useHostAuth === false) { + expect(proposal).toHaveBeenCalled(); + expect(preparation).not.toHaveBeenCalled(); + expect(saved?.errorCode).toBe("ai_connection_pool_exhausted"); + } else { + expect(proposal).not.toHaveBeenCalled(); + expect(saved?.contextSnapshot?.aiRouterSelection).toBeUndefined(); + expect(saved?.error).toContain("fixture stopped before provider configuration"); + expect(preparation).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ config: expect.objectContaining({ + aiBinding: undefined, sourceWorkspaceConfig: expect.objectContaining(configured), + }) })); + } + } finally { + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agent.id)); + await heartbeat.drainActiveRunExecutions(); + preparation.mockRestore(); + proposal.mockImplementation(defaultProposal); + await instanceSettingsService(db).updateExperimental(original); + } + }, 30_000); + it("admits a healthy pool member when the saved Grok connection is held", async () => { const original = await instanceSettingsService(db).getExperimental(); await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 028535af21a..84fefd182f4 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -25,7 +25,7 @@ import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; -import { selectHeartbeatEnvironment, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js"; +import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js"; import { environmentService } from "../services/environments.js"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { buildExecutionWorkspaceAdapterConfig } from "../services/execution-workspace-policy.js"; @@ -41,6 +41,21 @@ function deferred() { const skillFailures = vi.hoisted(() => new Map()); const skillWaits = vi.hoisted(() => new Map Promise>()); +describe("SSH authentication before account routing", () => { + it.each([ + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "agent", expected: "host" }, + { agentDefaultEnvironmentId: null, lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "instance", expected: "host" }, + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: "sandbox", executionMode: "any", expectedId: "sandbox", expected: null }, + { agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "kubernetes", expectedId: null, expected: null }, + ] as const)("honors environment precedence and forced execution policy: %j", async value => { + const get = vi.fn(async () => ({ defaultEnvironmentId: "instance", general: { executionMode: value.executionMode } })); + const getById = vi.fn(async (id: string) => ({ id, driver: id === "sandbox" ? "sandbox" : "ssh", config: {} } as Environment)); + expect(await resolveSshAuthenticationBeforeRouting({ ...value, instanceSettings: { get }, environmentsSvc: { getById } })).toBe(value.expected); + if (value.expectedId) expect(getById).toHaveBeenCalledExactlyOnceWith(value.expectedId); + else expect(getById).not.toHaveBeenCalled(); + }); +}); + describe("Runner home mode during recovery preparation", () => { const fingerprint = (effectiveAdapterConfig: Record) => buildEffectiveRunSessionConfigMetadata({ adapterType: "paperclip_runner", effectiveAdapterConfig, agentRuntimeConfig: {}, issueOverrides: null, diff --git a/server/src/__tests__/kimi-connection-probe.test.ts b/server/src/__tests__/kimi-connection-probe.test.ts index b7c05ae1118..a7a89991e94 100644 --- a/server/src/__tests__/kimi-connection-probe.test.ts +++ b/server/src/__tests__/kimi-connection-probe.test.ts @@ -24,7 +24,7 @@ vi.mock("node:child_process", async importOriginal => { spawn: (file: string, args: string[], options: Parameters[2]) => { if (file !== "ssh") return actual.spawn(file, args, options); mocks.wire.push({ file, args, stdin: true }); - return actual.spawn(process.execPath, ["-e", 'process.stdout.write(require("node:fs").readFileSync(0,"utf8"))'], options); + return actual.spawn(process.execPath, ["-e", 'const wire = require("node:fs").readFileSync(0); process.stdout.write(wire.subarray(wire.indexOf(10) + 1))'], options); }, }; }); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index c7e91e9cc2f..8892fe8ddc2 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3391,7 +3391,7 @@ export function agentRoutes( }); type = selection.adapterType; adapterConfigForTest = { ...selection.adapterConfig, - ...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" && requestedEnvironment.config.useHostAuth !== false ? { homeMode: "preconfigured" } : {}) }; + ...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" ? { homeMode: requestedEnvironment.config.useHostAuth === false ? "isolated" : "preconfigured" } : {}) }; if (requestedEnvironment) await assertAdapterTestEnvironmentForCompany(companyId, requestedEnvironment.id, type); const adapter = requireServerAdapter(type); if (!retainsSavedExecution) await assertSelectableAdapterType(type, adapterConfigForTest, companyId); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 32481819937..5374759701e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -58,7 +58,7 @@ import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery. import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability"; import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js"; import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; -import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; +import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting } from "./heartbeat/environment-selection.js"; import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js"; import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js"; @@ -2666,8 +2666,17 @@ export function heartbeatService( if ("homeMode" in homeConfig) config = { ...config, homeMode: homeConfig.homeMode }; } const persistedRouterPoolId = routerHasPersistedInput ? readNonEmptyString(parseObject(context.aiRouterSelection).poolId) : null; + const sshAuthentication = routerHasPersistedInput ? null : await resolveSshAuthenticationBeforeRouting({ + agentDefaultEnvironmentId: agent.defaultEnvironmentId, + lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review" + ? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, { includeEnvironmentId: true })?.environmentId ?? null : null, + environmentsSvc, instanceSettings, + }); + if (agent.adapterType === "paperclip_runner" && sshAuthentication) { + config = { ...config, homeMode: sshAuthentication === "host" ? "preconfigured" : "isolated" }; + } const preconfiguredHome = agent.adapterType === "paperclip_runner" && config.homeMode === "preconfigured"; - const requestedAiBinding = preconfiguredHome ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; + const requestedAiBinding = preconfiguredHome || sshAuthentication === "host" ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined; let aiBinding = requestedAiBinding?.mode === "router" ? undefined : requestedAiBinding; const originalAiIssueOverrides = issueAssigneeOverrides; if (requestedAiBinding?.mode === "router") { diff --git a/server/src/services/heartbeat/environment-selection.ts b/server/src/services/heartbeat/environment-selection.ts index ab0581c87f0..78f9e922b82 100644 --- a/server/src/services/heartbeat/environment-selection.ts +++ b/server/src/services/heartbeat/environment-selection.ts @@ -3,7 +3,7 @@ import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./e import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import { readGitCacheExpiration } from "../git-workspace-restoration.js"; -import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared"; +import type { ExecutionWorkspace, InstanceSettings, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared"; import { parseObject } from "../../adapters/utils.js"; import { logger } from "../../middleware/logger.js"; import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; @@ -39,6 +39,23 @@ import { resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspa type Issue = typeof issues.$inferSelect; +export async function resolveSshAuthenticationBeforeRouting(input: { + agentDefaultEnvironmentId: string | null; + lowTrustIssueEnvironmentId: string | null; + environmentsSvc: Pick, "getById">; + instanceSettings: { get(): Promise<{ + defaultEnvironmentId: InstanceSettings["defaultEnvironmentId"]; + general: Pick; + }> }; +}): Promise<"host" | "managed" | null> { + const settings = await input.instanceSettings.get(); + if (isExecutionForcedToKubernetes({ executionMode: settings.general.executionMode })) return null; + const id = input.lowTrustIssueEnvironmentId ?? input.agentDefaultEnvironmentId ?? settings.defaultEnvironmentId; + if (!id) return null; + const environment = await input.environmentsSvc.getById(id); + return environment?.driver === "ssh" ? environment.config.useHostAuth === false ? "managed" : "host" : null; +} + /** Selects policy and reuse inputs; lease acquisition and cleanup belong to the executor. */ export interface HeartbeatEnvironmentSelectionInput { run: Pick { }); it("still delivers every byte when SSH consumes the upload", async () => { - await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst hash = require('node:crypto').createHash('sha256');\nprocess.stdin.on('data', chunk => hash.update(chunk));\nprocess.stdin.on('end', () => process.stdout.write(hash.digest('hex')));\n`); + await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst chunks = [];\nprocess.stdin.on('data', chunk => chunks.push(chunk));\nprocess.stdin.on('end', () => { const wire = Buffer.concat(chunks); const end = wire.indexOf(10); if (end < 0) process.exit(1); process.stdout.write(require('node:crypto').createHash('sha256').update(wire.subarray(end + 1)).digest('hex')); });\n`); const stdin = "synthetic upload bytes\0".repeat(32_768); const result = await createRunner().execute({ command: "node", args: ["unused"], stdin, timeoutMs: 5000 }); expect(result.exitCode).toBe(0); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 2623962bee4..7fbe53f120e 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -392,7 +392,7 @@ "packages/shared/src/types/decision.ts": "b0d40077d481ce6180de6b02bad6df5bb0fbf977", "packages/shared/src/types/document-annotation.ts": "7769a42f1ccd707a2259774e7fe6aa536b5d183f", "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", - "packages/shared/src/types/environment.ts": "a380b9ce9d2d1763011e9adaa58615a3ef1e56c5", + "packages/shared/src/types/environment.ts": "16b9d9598abfa2583602181b44483b0fa24d5bc2", "packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b", "packages/shared/src/types/execution-projection.ts": "e72e641602b296f6d3e16d07700629df3039a7b1", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1",