Preserve SSH conversations and select authentication before pools

Keep only account-scoped Codex conversation and native index state between in-place SSH turns. Refresh current credentials and managed files for each turn, copy authentication back before cleanup, and start a named fresh session when saved state is unavailable or the account changes.

Resolve SSH authentication before pool selection and native home selection. Preserve admitted native recovery inputs. Update transport test fixtures for the stdin command envelope and refresh the committed SDK source pins.

Validated 164 execution/routing/probe tests, 91 CI regression tests, 58 SDK support tests, and Codex, Grok and server typechecks. Physical shell tests cover two turns, refresh, account changes, old saved parameters, and failure cleanup.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-11-12 17:42:03 -08:00
1 parent 8490bf9df2
commit 03e654e870
14 files changed
+293 -18

No files matched your search

+14
View File
@@ -459,6 +459,20 @@ Environment tests use the selected account and remove their temporary homes
after probing. A disconnected target can prevent remote cleanup; the run reports
the resulting failure rather than silently selecting another account.
Codex CLI runs in an in-place SSH workspace keep conversation files and native
SQLite indexes in a private remote home scoped to the company, agent, AI account,
and conversation. The home path stays stable across turns and token refreshes.
Each turn installs the current managed files, copies refreshed authentication
back, and removes everything except conversation state, including on provider
failure. Credentials and managed MCP configuration do not remain there between
turns. Switching accounts selects a different home and starts a new conversation.
The cache lives below the environment's remote workspace root. Removing that
root also removes the retained conversation state.
Older SSH host-auth sessions keep their existing remote home. A managed SSH
session saved before conversation retention has no retained state reference and
starts a fresh conversation instead of reusing an unavailable thread. This reset
is recorded as `codex_ssh_session_state_unavailable`; existing saved files remain.
ACP SSH sessions use the existing remote process bridge and session reuse
lifecycle. A compatible session can retain its private remote home; workspace
sync and credential refresh run at turn teardown, and the session disposal hook
@@ -1,4 +1,6 @@
import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -27,7 +29,7 @@ const {
prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })),
restoreWorkspaceFromSshExecution: vi.fn(async () => undefined),
runSshCommand: vi.fn(async (_spec?: unknown, _command?: string) => ({ stdout: Buffer.from("{}").toString("base64"), stderr: "" })),
syncDirectoryToSsh: vi.fn(async () => undefined),
syncDirectoryToSsh: vi.fn(async (_input?: { localDir: string; remoteDir: string }) => undefined),
startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({
env: {
PAPERCLIP_API_URL: "http://127.0.0.1:4310",
@@ -88,6 +90,109 @@ describe("codex remote execution", () => {
}
});
it.each([false, true])("retains a managed SSH Codex conversation across turns and removes credentials (second turn throws=%s)", async fails => {
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-ssh-resume-"));
cleanupDirs.push(root);
const workspace = path.join(root, "workspace"), home = path.join(root, "account"), secondHome = path.join(root, "next-invocation");
await mkdir(workspace); await mkdir(home); await mkdir(secondHome);
const auth = (account: string, marker: string, last_refresh: string) => JSON.stringify({ tokens: {
account_id: account, access_token: `fixture-access-${marker}`, refresh_token: `fixture-refresh-${marker}`, id_token: `fixture-id-${marker}`,
}, last_refresh });
const credential = auth("selected", "before", "2000-01-01T00:00:00Z");
const refreshed = auth("selected", "after", new Date().toISOString());
const otherCredential = auth("other", "other", "2000-01-01T00:00:00Z");
await writeFile(path.join(home, "auth.json"), credential);
await writeFile(path.join(secondHome, "auth.json"), credential);
const executeShell = promisify(execFile);
const originalRun = runChildProcess.getMockImplementation()!;
let firstRemoteHome: string | undefined;
let otherRemoteHome: string | undefined;
const privateRoots: string[] = [];
let calls = 0;
runSshCommand.mockImplementation(async (_spec, command) => {
const result = await executeShell("sh", ["-c", command!]);
if (command!.includes("mktemp -d")) privateRoots.push(result.stdout.trim());
return result;
});
syncDirectoryToSsh.mockImplementation(async input => { await cp(input!.localDir, input!.remoteDir, { recursive: true }); });
runChildProcess.mockImplementation(async (...args: unknown[]) => {
const env = (args[3] as { env: Record<string, string> }).env;
const remoteHome = env.CODEX_HOME!;
calls++;
expect(await readFile(path.join(remoteHome, "auth.json"), "utf8")).toBe(calls === 1 ? credential : calls === 2 ? refreshed : otherCredential);
if (calls === 1) {
firstRemoteHome = remoteHome;
await mkdir(path.join(remoteHome, "sessions"));
await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "fixture conversation");
await writeFile(path.join(remoteHome, "state_5.sqlite"), "fixture native index");
await writeFile(path.join(remoteHome, "auth.json"), refreshed);
} else if (calls === 2) {
expect(remoteHome).toBe(firstRemoteHome);
expect(args[2]).toContain("resume");
expect(args[2]).toContain("ssh-thread");
expect(await readFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation");
expect(await readFile(path.join(remoteHome, "state_5.sqlite"), "utf8")).toBe("fixture native index");
if (fails) throw new Error("fixture provider failed");
} else {
if (calls === 3) otherRemoteHome = remoteHome;
expect(remoteHome).not.toBe(firstRemoteHome);
expect(args[2]).not.toContain("resume");
await expect(readFile(path.join(remoteHome, "sessions", "rollout.jsonl"))).rejects.toThrow("ENOENT");
if (calls === 3) {
await mkdir(path.join(remoteHome, "sessions"));
await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "other conversation");
}
}
return { exitCode: 0, signal: null, timedOut: false, stdout: JSON.stringify({ type: "thread.started", thread_id: calls < 3 ? "ssh-thread" : "other-thread" })+'\n{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', stderr: "", pid: 123, startedAt: new Date().toISOString() };
});
const ctx = {
agent: { id: "agent-1", companyId: "company-1", name: "Selected agent", adapterType: "codex_local", adapterConfig: {} },
config: { engine: "cli", command: "codex", managedAiConnection: { identity: "generation-one", sessionIdentity: "selected-account", provider: "openai", method: "subscription" }, env: { CODEX_HOME: home, OPENAI_API_KEY: "" } },
context: { paperclipWorkspace: { cwd: workspace, source: "project_primary" } },
executionTarget: { kind: "remote" as const, transport: "ssh" as const, remoteCwd: workspace,
spec: { useHostAuth: false, host: "127.0.0.1", port: 2222, username: "fixture", remoteCwd: workspace, remoteWorkspacePath: workspace, privateKey: null, knownHosts: null, strictHostKeyChecking: true },
workspaceRealization: { mode: "in_place" as const, authoritativeRoot: workspace, pathAliases: [], outboundRestorePaths: [] } }, onLog: async () => {},
};
try {
const first = await execute({ ...ctx, runId: "first", runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: "task-one" } });
expect(first.sessionId).toBe("ssh-thread");
expect(await readFile(path.join(home, "auth.json"), "utf8")).toBe(refreshed);
await writeFile(path.join(secondHome, "auth.json"), refreshed);
expect(await readdir(firstRemoteHome!)).toEqual(expect.arrayContaining(["sessions", "state_5.sqlite"]));
expect(await readdir(firstRemoteHome!)).not.toContain("auth.json");
const second = execute({ ...ctx, runId: "second", config: { ...ctx.config,
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "generation-two" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } });
if (fails) await expect(second).rejects.toThrow("fixture provider failed");
else expect((await second).sessionId).toBe("ssh-thread");
expect(calls).toBe(2);
expect((await readdir(firstRemoteHome!)).sort()).toEqual(["sessions", "state_5.sqlite"]);
expect((await stat(firstRemoteHome!)).mode & 0o077).toBe(0);
if (!fails) {
await writeFile(path.join(secondHome, "auth.json"), otherCredential);
const switched = await execute({ ...ctx, runId: "other-account", config: { ...ctx.config,
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } });
expect(switched.sessionId).toBe("other-thread");
expect(switched.sessionParams?.sshCodexAccountScope).not.toBe(first.sessionParams?.sshCodexAccountScope);
expect(switched.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } });
expect(await readdir(otherRemoteHome!)).not.toContain("auth.json");
expect(await readFile(path.join(firstRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation");
const oldShape = { sessionId: switched.sessionId!, cwd: workspace, remoteExecution: switched.sessionParams!.remoteExecution };
const legacy = await execute({ ...ctx, runId: "old-saved-shape", config: { ...ctx.config,
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
runtime: { sessionId: switched.sessionId!, sessionParams: oldShape, sessionDisplayId: switched.sessionId!, taskKey: "task-one" } });
expect(legacy.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } });
expect(await readFile(path.join(otherRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("other conversation");
}
for (const dir of privateRoots) await expect(stat(dir)).rejects.toThrow("ENOENT");
} finally {
runChildProcess.mockImplementation(originalRun);
runSshCommand.mockReset().mockResolvedValue({ stdout: Buffer.from("{}").toString("base64"), stderr: "" });
syncDirectoryToSsh.mockReset().mockResolvedValue(undefined);
}
});
it("stages only the selected agent subscription and returns SSH refreshes to that account", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-selected-ssh-"));
cleanupDirs.push(root);
@@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url";
import { inferOpenAiCompatibleBiller, resolveManagedOpenAiBilling, type AdapterExecutionContext, type AdapterExecutionResult } from "@paperclipai/adapter-utils";
import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js";
import { copyBackCodexAuth } from "./codex-auth-copyback.js";
import { prepareCodexSshSessionState } from "./ssh-session-state.js";
import {
ensureCodexAuthCacheEntryDir,
isCodexAuthCacheEnabled,
@@ -783,6 +784,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
// here so the outer `finally` can remove it on every exit path (teardown and
// error), never only the happy path.
let stagedCodexHomeDir: string | null = null;
let sshSessionState: Awaited<ReturnType<typeof prepareCodexSshSessionState>> | null = null;
let observabilityProfile: Awaited<ReturnType<typeof prepareCodexObservabilityProfile>> | null = null;
let observabilityPreparationFailed = false;
try {
@@ -894,9 +896,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
// generic `restore` seam per asset before destroying the sandbox.
// Target is the shared symlink SOURCE (what managed homes point
// `auth.json` at), not the in-sandbox symlink.
restore: async ({ assetDir, readFile }) =>
void (await copyBackCodexAuth({
readSandboxAuth: () => readFile(path.posix.join(assetDir, "auth.json")),
restore: async ({ assetDir, readFile }) => {
try { await copyBackCodexAuth({
readSandboxAuth: () => readFile(path.posix.join(sshSessionState?.home ?? assetDir, "auth.json")),
hostAuthPath: path.join(config.managedAiConnection ? effectiveCodexHome : resolveSharedCodexHomeDir(process.env), "auth.json"),
log: (line) => onLog("stdout", `${line}\n`),
// Additive cache write (sandbox to host): also cache the
@@ -907,7 +909,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
resolveCacheEntryPath: config.managedAiConnection ? undefined : (accountId) =>
ensureCodexAuthCacheEntryDir(process.env, accountId, agent.companyId),
env: process.env,
})),
}); } finally { await sshSessionState?.cleanup(); }
},
// No `exclude` denylist: `stagedCodexHomeDir` already contains
// ONLY the allowlisted files (auth/config/skills), so there is
// nothing to filter out.
@@ -928,10 +931,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
let paperclipBridge: Awaited<ReturnType<typeof startAdapterExecutionTargetPaperclipBridge>> = null;
let executionError: unknown = null;
try {
const remoteCodexHome = executionTargetIsRemote && !useSshHostAuth
let remoteCodexHome = executionTargetIsRemote && !useSshHostAuth
? preparedExecutionTargetRuntime?.assetDirs.home ??
path.posix.join(effectiveExecutionCwd, ".paperclip-runtime", "codex", "home")
: null;
if (executionTarget?.kind === "remote" && executionTarget.transport === "ssh" && executionTarget.spec.useHostAuth === false
&& config.managedAiConnection && targetWorkspaceRealization?.mode === "in_place" && remoteCodexHome) {
const account = parseObject(config.managedAiConnection);
const identity = asString(account.sessionIdentity, asString(account.identity, ""));
if (!identity) throw new Error("Managed SSH Codex execution requires an account identity");
sshSessionState = await prepareCodexSshSessionState({
remoteRoot: executionTarget.spec.remoteWorkspacePath, companyId: agent.companyId, agentId: agent.id,
accountIdentity: identity, executionIdentity: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget),
runId, saved: parseObject(runtime.sessionParams), stagedHome: remoteCodexHome,
run: async command => {
const result = await runAdapterExecutionTargetShellCommand(runId, runtimeExecutionTarget, command, { cwd: effectiveExecutionCwd, env: {}, timeoutSec: 15 });
if (result.exitCode !== 0 || result.timedOut) throw new Error("Cannot prepare or clean up SSH Codex conversation state");
},
});
remoteCodexHome = sshSessionState.home;
}
await emitSandboxAuthPrecedenceWarningIfNeeded({
runId,
target: runtimeExecutionTarget,
@@ -1138,6 +1157,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
const runtimeRemoteExecution = parseObject(runtimeSessionParams.remoteExecution);
const canResumeSession =
runtimeSessionId.length > 0 &&
(!sshSessionState || sshSessionState.canResume) &&
(runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(effectiveExecutionCwd)) &&
adapterExecutionTargetSessionMatches(runtimeRemoteExecution, runtimeExecutionTarget);
const codexTransientFallbackMode = readCodexTransientFallbackMode(context);
@@ -1525,7 +1545,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
};
}
const canFallbackToRuntimeSession = !isRetry && !forceFreshSession;
const canFallbackToRuntimeSession = !isRetry && !forceFreshSession && (!sshSessionState || sshSessionState.canResume);
const resolvedSessionId =
attempt.parsed.sessionId ??
(canFallbackToRuntimeSession ? (runtimeSessionId ?? runtime.sessionId ?? null) : null);
@@ -1533,6 +1553,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
? ({
sessionId: resolvedSessionId,
cwd: effectiveExecutionCwd,
...(sshSessionState ? { sshCodexConversationId: sshSessionState.conversationId, sshCodexAccountScope: sshSessionState.scope } : {}),
...(executionTargetIsRemote
? {
remoteExecution: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget),
@@ -1636,6 +1657,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
resultJson: {
stdout: attempt.proc.stdout,
stderr: attempt.proc.stderr,
...(sshSessionState && runtimeSessionId && !sshSessionState.canResume
? { sessionReset: { code: "codex_ssh_session_state_unavailable" } } : {}),
...(errorFamily ? { errorFamily } : {}),
...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
@@ -0,0 +1,41 @@
import { createHash } from "node:crypto";
import path from "node:path";
import { shellQuote } from "@paperclipai/adapter-utils/ssh";
const STATE_ONLY = `for entry in "$home"/* "$home"/.[!.]* "$home"/..?*; do
[ -e "$entry" ] || [ -L "$entry" ] || continue
if [ ! -L "$entry" ]; then
case "\${entry##*/}" in
sessions|archived_sessions|session_index.jsonl|history.jsonl|state_*.sqlite|state_*.sqlite-*|thread_history_*.sqlite|thread_history_*.sqlite-*) continue ;;
esac
fi
rm -rf -- "$entry"
done`;
export async function prepareCodexSshSessionState(input: {
remoteRoot: string;
companyId: string;
agentId: string;
accountIdentity: string;
executionIdentity: unknown;
runId: string;
saved: Record<string, unknown>;
stagedHome: string;
run: (command: string) => Promise<void>;
}) {
const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
const scope = hash([input.companyId, input.agentId, input.accountIdentity, input.executionIdentity]);
const savedId = input.saved.sshCodexConversationId;
const canResume = input.saved.sshCodexAccountScope === scope && typeof savedId === "string" && /^[a-f0-9]{64}$/.test(savedId);
const conversationId = canResume ? savedId : hash(input.runId);
const home = path.posix.join(input.remoteRoot, ".paperclip-runtime", "codex", "conversations", scope, conversationId);
const prefix = `set -eu\numask 077\nhome=${shellQuote(home)}\n[ ! -L "$home" ]\n`;
const cleanup = () => input.run(`${prefix}if [ -d "$home" ]; then\n${STATE_ONLY}\nfi`);
try {
await input.run(`${prefix}mkdir -p "$home"\nchmod 700 "$home"\n${STATE_ONLY}\ncp -RP ${shellQuote(input.stagedHome + "/.")} "$home/"`);
} catch (error) {
await cleanup();
throw error;
}
return { home, scope, conversationId, canResume, cleanup };
}
@@ -64,7 +64,10 @@ vi.mock("@paperclipai/adapter-utils/execution-target", () => ({
prepareAdapterExecutionTargetRuntime: (...args: unknown[]) =>
(mocks.prepareRuntimeMock as (...args: unknown[]) => unknown)(...args),
readAdapterExecutionTarget: (input: { executionTarget?: unknown }) => input.executionTarget ??
(mocks.state.isRemote ? { kind: "remote", transport: "ssh" } : { kind: "local" }),
(mocks.state.isRemote ? { kind: "remote", transport: "ssh", remoteCwd: "/remote/workspace", spec: {
host: "fixture.invalid", port: 22, username: "fixture", remoteCwd: "/remote/workspace", remoteWorkspacePath: "/remote/workspace",
privateKey: null, knownHosts: null, strictHostKeyChecking: true,
} } : { kind: "local" }),
resolveAdapterExecutionTargetCommandForLogs: (...args: unknown[]) =>
(mocks.resolveCommandForLogsMock as (...args: unknown[]) => unknown)(...args),
resolveAdapterExecutionTargetTimeoutSec: (_target: unknown, timeoutSec: number) => timeoutSec,
@@ -27,6 +27,8 @@ import { legacyExecutionNeedsReconciliation } from "../services/legacy-execution
import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
import { dotRunnerBroker } from "../services/dot-runner-broker.js";
import { environmentService } from "../services/environments.js";
import * as executionConfiguration from "../services/heartbeat/execution-configuration.js";
const captureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../sentry.js", async (importOriginal) => ({
@@ -446,6 +448,52 @@ describe("durable, authorized connection routing", () => {
expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: String(selected.runtimeConfig.model), taskSessionParams: { __paperclipConfiguredModel: selected.runtimeConfig.model, __paperclipConfigFingerprint: previous.fingerprint, __paperclipConfigFingerprintVersion: previous.version, __paperclipConfigCategoryFingerprints: previous.categoryFingerprints }, configMetadata: current, compatibleConfigMetadata }).reset).toBe(false);
} finally { await before.cleanup(); await after.cleanup(); }
});
it.each([
{ useHostAuth: undefined, nativeHomeMode: undefined }, { useHostAuth: true, nativeHomeMode: undefined },
{ useHostAuth: false, nativeHomeMode: undefined }, { useHostAuth: false, nativeHomeMode: "preconfigured" },
])("applies SSH authentication before selecting a busy pool (%j)", async ({ useHostAuth, nativeHomeMode }) => {
const original = await instanceSettingsService(db).getExperimental();
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, enableNativeRunner: true, enableEnvironments: true, enableWorktreeRunExecution: true });
const pool = await makePool({ members: [members[0]!] });
const environment = await environmentService(db).create(companyId, { name: `SSH routing fixture ${useHostAuth} ${nativeHomeMode}`, driver: "ssh", status: "active", envVars: {}, config: {
host: "ssh.example.test", username: "fixture", remoteWorkspacePath: "/workspace", ...(useHostAuth === undefined ? {} : { useHostAuth }),
} });
const configured = { provider: "acpx", acpxAgent: "claude", model: "fixture-host-model", ...(nativeHomeMode ? { homeMode: nativeHomeMode } : {}) };
const [agent] = await db.insert(agents).values({ companyId, name: "SSH routing fixture", status: "idle", adapterType: "paperclip_runner",
adapterConfig: configured, defaultEnvironmentId: environment.id,
runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id }, heartbeat: { enabled: false, wakeOnDemand: true } } }).returning();
const [issue] = await db.insert(issues).values({ companyId, title: "SSH pool routing fixture", status: "todo", assigneeAgentId: agent.id, responsibleUserId: "alice", createdByUserId: "alice" }).returning();
proposal.mockClear();
proposal.mockImplementation(async () => ({ kind: "exhausted", retryAt: new Date(Date.now() + 60_000).toISOString(), skipped: {} }));
const preparation = vi.spyOn(executionConfiguration, "prepareHeartbeatExecutionConfiguration").mockRejectedValue(new Error("fixture stopped before provider configuration"));
const heartbeat = heartbeatService(db, { pluginWorkerManager: worker, runtimeEnv: {} });
try {
const run = await heartbeat.wakeup(agent.id, { source: "assignment", reason: "issue_assigned", triggerDetail: "system",
payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, responsibleUserId: "alice" } });
expect(run).not.toBeNull();
await heartbeat.drainActiveRunExecutions();
const saved = await heartbeat.getRun(run!.id);
if (useHostAuth === false) {
expect(proposal).toHaveBeenCalled();
expect(preparation).not.toHaveBeenCalled();
expect(saved?.errorCode).toBe("ai_connection_pool_exhausted");
} else {
expect(proposal).not.toHaveBeenCalled();
expect(saved?.contextSnapshot?.aiRouterSelection).toBeUndefined();
expect(saved?.error).toContain("fixture stopped before provider configuration");
expect(preparation).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ config: expect.objectContaining({
aiBinding: undefined, sourceWorkspaceConfig: expect.objectContaining(configured),
}) }));
}
} finally {
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agent.id));
await heartbeat.drainActiveRunExecutions();
preparation.mockRestore();
proposal.mockImplementation(defaultProposal);
await instanceSettingsService(db).updateExperimental(original);
}
}, 30_000);
it("admits a healthy pool member when the saved Grok connection is held", async () => {
const original = await instanceSettingsService(db).getExperimental();
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true,
@@ -25,7 +25,7 @@ import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
import * as sentry from "../sentry.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { selectHeartbeatEnvironment, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js";
import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js";
import { environmentService } from "../services/environments.js";
import { executionWorkspaceService } from "../services/execution-workspaces.js";
import { buildExecutionWorkspaceAdapterConfig } from "../services/execution-workspace-policy.js";
@@ -41,6 +41,21 @@ function deferred<T>() {
const skillFailures = vi.hoisted(() => new Map<string, string>());
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
describe("SSH authentication before account routing", () => {
it.each([
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "agent", expected: "host" },
{ agentDefaultEnvironmentId: null, lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "instance", expected: "host" },
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: "sandbox", executionMode: "any", expectedId: "sandbox", expected: null },
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "kubernetes", expectedId: null, expected: null },
] as const)("honors environment precedence and forced execution policy: %j", async value => {
const get = vi.fn(async () => ({ defaultEnvironmentId: "instance", general: { executionMode: value.executionMode } }));
const getById = vi.fn(async (id: string) => ({ id, driver: id === "sandbox" ? "sandbox" : "ssh", config: {} } as Environment));
expect(await resolveSshAuthenticationBeforeRouting({ ...value, instanceSettings: { get }, environmentsSvc: { getById } })).toBe(value.expected);
if (value.expectedId) expect(getById).toHaveBeenCalledExactlyOnceWith(value.expectedId);
else expect(getById).not.toHaveBeenCalled();
});
});
describe("Runner home mode during recovery preparation", () => {
const fingerprint = (effectiveAdapterConfig: Record<string, unknown>) => buildEffectiveRunSessionConfigMetadata({
adapterType: "paperclip_runner", effectiveAdapterConfig, agentRuntimeConfig: {}, issueOverrides: null,
@@ -24,7 +24,7 @@ vi.mock("node:child_process", async importOriginal => {
spawn: (file: string, args: string[], options: Parameters<typeof actual.spawn>[2]) => {
if (file !== "ssh") return actual.spawn(file, args, options);
mocks.wire.push({ file, args, stdin: true });
return actual.spawn(process.execPath, ["-e", 'process.stdout.write(require("node:fs").readFileSync(0,"utf8"))'], options);
return actual.spawn(process.execPath, ["-e", 'const wire = require("node:fs").readFileSync(0); process.stdout.write(wire.subarray(wire.indexOf(10) + 1))'], options);
},
};
});
+1 -1
View File
@@ -3391,7 +3391,7 @@ export function agentRoutes(
});
type = selection.adapterType;
adapterConfigForTest = { ...selection.adapterConfig,
...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" && requestedEnvironment.config.useHostAuth !== false ? { homeMode: "preconfigured" } : {}) };
...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" ? { homeMode: requestedEnvironment.config.useHostAuth === false ? "isolated" : "preconfigured" } : {}) };
if (requestedEnvironment) await assertAdapterTestEnvironmentForCompany(companyId, requestedEnvironment.id, type);
const adapter = requireServerAdapter(type);
if (!retainsSavedExecution) await assertSelectableAdapterType(type, adapterConfigForTest, companyId);
+11 -2
View File
@@ -58,7 +58,7 @@ import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.
import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability";
import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js";
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting } from "./heartbeat/environment-selection.js";
import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js";
import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js";
@@ -2666,8 +2666,17 @@ export function heartbeatService(
if ("homeMode" in homeConfig) config = { ...config, homeMode: homeConfig.homeMode };
}
const persistedRouterPoolId = routerHasPersistedInput ? readNonEmptyString(parseObject(context.aiRouterSelection).poolId) : null;
const sshAuthentication = routerHasPersistedInput ? null : await resolveSshAuthenticationBeforeRouting({
agentDefaultEnvironmentId: agent.defaultEnvironmentId,
lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review"
? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, { includeEnvironmentId: true })?.environmentId ?? null : null,
environmentsSvc, instanceSettings,
});
if (agent.adapterType === "paperclip_runner" && sshAuthentication) {
config = { ...config, homeMode: sshAuthentication === "host" ? "preconfigured" : "isolated" };
}
const preconfiguredHome = agent.adapterType === "paperclip_runner" && config.homeMode === "preconfigured";
const requestedAiBinding = preconfiguredHome ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
const requestedAiBinding = preconfiguredHome || sshAuthentication === "host" ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
let aiBinding = requestedAiBinding?.mode === "router" ? undefined : requestedAiBinding;
const originalAiIssueOverrides = issueAssigneeOverrides;
if (requestedAiBinding?.mode === "router") {
@@ -3,7 +3,7 @@ import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./e
import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db";
import { and, eq } from "drizzle-orm";
import { readGitCacheExpiration } from "../git-workspace-restoration.js";
import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared";
import type { ExecutionWorkspace, InstanceSettings, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared";
import { parseObject } from "../../adapters/utils.js";
import { logger } from "../../middleware/logger.js";
import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
@@ -39,6 +39,23 @@ import { resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspa
type Issue = typeof issues.$inferSelect;
export async function resolveSshAuthenticationBeforeRouting(input: {
agentDefaultEnvironmentId: string | null;
lowTrustIssueEnvironmentId: string | null;
environmentsSvc: Pick<ReturnType<typeof environmentService>, "getById">;
instanceSettings: { get(): Promise<{
defaultEnvironmentId: InstanceSettings["defaultEnvironmentId"];
general: Pick<InstanceSettings["general"], "executionMode">;
}> };
}): Promise<"host" | "managed" | null> {
const settings = await input.instanceSettings.get();
if (isExecutionForcedToKubernetes({ executionMode: settings.general.executionMode })) return null;
const id = input.lowTrustIssueEnvironmentId ?? input.agentDefaultEnvironmentId ?? settings.defaultEnvironmentId;
if (!id) return null;
const environment = await input.environmentsSvc.getById(id);
return environment?.driver === "ssh" ? environment.config.useHostAuth === false ? "managed" : "host" : null;
}
/** Selects policy and reuse inputs; lease acquisition and cleanup belong to the executor. */
export interface HeartbeatEnvironmentSelectionInput {
run: Pick<typeof heartbeatRuns.$inferSelect,
@@ -102,7 +102,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
const useSshHostAuth = selectedEnvironmentForConfig?.driver === "ssh" && selectedEnvironmentForConfig.config.useHostAuth !== false && !persistedNativeExecutionInput;
const sourceWorkspaceConfig = agent.adapterType === "paperclip_runner"
? { ...input.config.sourceWorkspaceConfig,
homeMode: bindHeartbeatRunnerHomeMode({ ...parseObject(agent.adapterConfig), ...(useSshHostAuth ? { homeMode: "preconfigured" } : {}) }, persistedNativeExecutionInput).homeMode }
homeMode: bindHeartbeatRunnerHomeMode({ ...parseObject(agent.adapterConfig), ...(selectedEnvironmentForConfig?.driver === "ssh" && !persistedNativeExecutionInput ? { homeMode: useSshHostAuth ? "preconfigured" : "isolated" } : {}) }, persistedNativeExecutionInput).homeMode }
: input.config.sourceWorkspaceConfig;
const preconfiguredHome = agent.adapterType === "paperclip_runner" && sourceWorkspaceConfig.homeMode === "preconfigured";
if (preconfiguredHome && pinnedSkillTest) {
@@ -58,7 +58,7 @@ describe("native SSH deliverable output budget", () => {
});
it("still delivers every byte when SSH consumes the upload", async () => {
await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst hash = require('node:crypto').createHash('sha256');\nprocess.stdin.on('data', chunk => hash.update(chunk));\nprocess.stdin.on('end', () => process.stdout.write(hash.digest('hex')));\n`);
await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst chunks = [];\nprocess.stdin.on('data', chunk => chunks.push(chunk));\nprocess.stdin.on('end', () => { const wire = Buffer.concat(chunks); const end = wire.indexOf(10); if (end < 0) process.exit(1); process.stdout.write(require('node:crypto').createHash('sha256').update(wire.subarray(end + 1)).digest('hex')); });\n`);
const stdin = "synthetic upload bytes\0".repeat(32_768);
const result = await createRunner().execute({ command: "node", args: ["unused"], stdin, timeoutMs: 5000 });
expect(result.exitCode).toBe(0);
@@ -392,7 +392,7 @@
"packages/shared/src/types/decision.ts": "b0d40077d481ce6180de6b02bad6df5bb0fbf977",
"packages/shared/src/types/document-annotation.ts": "7769a42f1ccd707a2259774e7fe6aa536b5d183f",
"packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc",
"packages/shared/src/types/environment.ts": "a380b9ce9d2d1763011e9adaa58615a3ef1e56c5",
"packages/shared/src/types/environment.ts": "16b9d9598abfa2583602181b44483b0fa24d5bc2",
"packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b",
"packages/shared/src/types/execution-projection.ts": "e72e641602b296f6d3e16d07700629df3039a7b1",
"packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1",