mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 04:21:24 +01:00
Preserve SSH conversations and select authentication before pools
Keep only account-scoped Codex conversation and native index state between in-place SSH turns. Refresh current credentials and managed files for each turn, copy authentication back before cleanup, and start a named fresh session when saved state is unavailable or the account changes. Resolve SSH authentication before pool selection and native home selection. Preserve admitted native recovery inputs. Update transport test fixtures for the stdin command envelope and refresh the committed SDK source pins. Validated 164 execution/routing/probe tests, 91 CI regression tests, 58 SDK support tests, and Codex, Grok and server typechecks. Physical shell tests cover two turns, refresh, account changes, old saved parameters, and failure cleanup. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
8490bf9df2
commit
03e654e870
14 files changed
+293
-18
No files matched your search
@@ -459,6 +459,20 @@ Environment tests use the selected account and remove their temporary homes
|
||||
after probing. A disconnected target can prevent remote cleanup; the run reports
|
||||
the resulting failure rather than silently selecting another account.
|
||||
|
||||
Codex CLI runs in an in-place SSH workspace keep conversation files and native
|
||||
SQLite indexes in a private remote home scoped to the company, agent, AI account,
|
||||
and conversation. The home path stays stable across turns and token refreshes.
|
||||
Each turn installs the current managed files, copies refreshed authentication
|
||||
back, and removes everything except conversation state, including on provider
|
||||
failure. Credentials and managed MCP configuration do not remain there between
|
||||
turns. Switching accounts selects a different home and starts a new conversation.
|
||||
The cache lives below the environment's remote workspace root. Removing that
|
||||
root also removes the retained conversation state.
|
||||
Older SSH host-auth sessions keep their existing remote home. A managed SSH
|
||||
session saved before conversation retention has no retained state reference and
|
||||
starts a fresh conversation instead of reusing an unavailable thread. This reset
|
||||
is recorded as `codex_ssh_session_state_unavailable`; existing saved files remain.
|
||||
|
||||
ACP SSH sessions use the existing remote process bridge and session reuse
|
||||
lifecycle. A compatible session can retain its private remote home; workspace
|
||||
sync and credential refresh run at turn teardown, and the session disposal hook
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { mkdir, mkdtemp, readdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -27,7 +29,7 @@ const {
|
||||
prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })),
|
||||
restoreWorkspaceFromSshExecution: vi.fn(async () => undefined),
|
||||
runSshCommand: vi.fn(async (_spec?: unknown, _command?: string) => ({ stdout: Buffer.from("{}").toString("base64"), stderr: "" })),
|
||||
syncDirectoryToSsh: vi.fn(async () => undefined),
|
||||
syncDirectoryToSsh: vi.fn(async (_input?: { localDir: string; remoteDir: string }) => undefined),
|
||||
startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({
|
||||
env: {
|
||||
PAPERCLIP_API_URL: "http://127.0.0.1:4310",
|
||||
@@ -88,6 +90,109 @@ describe("codex remote execution", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("retains a managed SSH Codex conversation across turns and removes credentials (second turn throws=%s)", async fails => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-ssh-resume-"));
|
||||
cleanupDirs.push(root);
|
||||
const workspace = path.join(root, "workspace"), home = path.join(root, "account"), secondHome = path.join(root, "next-invocation");
|
||||
await mkdir(workspace); await mkdir(home); await mkdir(secondHome);
|
||||
const auth = (account: string, marker: string, last_refresh: string) => JSON.stringify({ tokens: {
|
||||
account_id: account, access_token: `fixture-access-${marker}`, refresh_token: `fixture-refresh-${marker}`, id_token: `fixture-id-${marker}`,
|
||||
}, last_refresh });
|
||||
const credential = auth("selected", "before", "2000-01-01T00:00:00Z");
|
||||
const refreshed = auth("selected", "after", new Date().toISOString());
|
||||
const otherCredential = auth("other", "other", "2000-01-01T00:00:00Z");
|
||||
await writeFile(path.join(home, "auth.json"), credential);
|
||||
await writeFile(path.join(secondHome, "auth.json"), credential);
|
||||
const executeShell = promisify(execFile);
|
||||
const originalRun = runChildProcess.getMockImplementation()!;
|
||||
let firstRemoteHome: string | undefined;
|
||||
let otherRemoteHome: string | undefined;
|
||||
const privateRoots: string[] = [];
|
||||
let calls = 0;
|
||||
runSshCommand.mockImplementation(async (_spec, command) => {
|
||||
const result = await executeShell("sh", ["-c", command!]);
|
||||
if (command!.includes("mktemp -d")) privateRoots.push(result.stdout.trim());
|
||||
return result;
|
||||
});
|
||||
syncDirectoryToSsh.mockImplementation(async input => { await cp(input!.localDir, input!.remoteDir, { recursive: true }); });
|
||||
runChildProcess.mockImplementation(async (...args: unknown[]) => {
|
||||
const env = (args[3] as { env: Record<string, string> }).env;
|
||||
const remoteHome = env.CODEX_HOME!;
|
||||
calls++;
|
||||
expect(await readFile(path.join(remoteHome, "auth.json"), "utf8")).toBe(calls === 1 ? credential : calls === 2 ? refreshed : otherCredential);
|
||||
if (calls === 1) {
|
||||
firstRemoteHome = remoteHome;
|
||||
await mkdir(path.join(remoteHome, "sessions"));
|
||||
await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "fixture conversation");
|
||||
await writeFile(path.join(remoteHome, "state_5.sqlite"), "fixture native index");
|
||||
await writeFile(path.join(remoteHome, "auth.json"), refreshed);
|
||||
} else if (calls === 2) {
|
||||
expect(remoteHome).toBe(firstRemoteHome);
|
||||
expect(args[2]).toContain("resume");
|
||||
expect(args[2]).toContain("ssh-thread");
|
||||
expect(await readFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation");
|
||||
expect(await readFile(path.join(remoteHome, "state_5.sqlite"), "utf8")).toBe("fixture native index");
|
||||
if (fails) throw new Error("fixture provider failed");
|
||||
} else {
|
||||
if (calls === 3) otherRemoteHome = remoteHome;
|
||||
expect(remoteHome).not.toBe(firstRemoteHome);
|
||||
expect(args[2]).not.toContain("resume");
|
||||
await expect(readFile(path.join(remoteHome, "sessions", "rollout.jsonl"))).rejects.toThrow("ENOENT");
|
||||
if (calls === 3) {
|
||||
await mkdir(path.join(remoteHome, "sessions"));
|
||||
await writeFile(path.join(remoteHome, "sessions", "rollout.jsonl"), "other conversation");
|
||||
}
|
||||
}
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout: JSON.stringify({ type: "thread.started", thread_id: calls < 3 ? "ssh-thread" : "other-thread" })+'\n{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}\n', stderr: "", pid: 123, startedAt: new Date().toISOString() };
|
||||
});
|
||||
const ctx = {
|
||||
agent: { id: "agent-1", companyId: "company-1", name: "Selected agent", adapterType: "codex_local", adapterConfig: {} },
|
||||
config: { engine: "cli", command: "codex", managedAiConnection: { identity: "generation-one", sessionIdentity: "selected-account", provider: "openai", method: "subscription" }, env: { CODEX_HOME: home, OPENAI_API_KEY: "" } },
|
||||
context: { paperclipWorkspace: { cwd: workspace, source: "project_primary" } },
|
||||
executionTarget: { kind: "remote" as const, transport: "ssh" as const, remoteCwd: workspace,
|
||||
spec: { useHostAuth: false, host: "127.0.0.1", port: 2222, username: "fixture", remoteCwd: workspace, remoteWorkspacePath: workspace, privateKey: null, knownHosts: null, strictHostKeyChecking: true },
|
||||
workspaceRealization: { mode: "in_place" as const, authoritativeRoot: workspace, pathAliases: [], outboundRestorePaths: [] } }, onLog: async () => {},
|
||||
};
|
||||
try {
|
||||
const first = await execute({ ...ctx, runId: "first", runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: "task-one" } });
|
||||
expect(first.sessionId).toBe("ssh-thread");
|
||||
expect(await readFile(path.join(home, "auth.json"), "utf8")).toBe(refreshed);
|
||||
await writeFile(path.join(secondHome, "auth.json"), refreshed);
|
||||
expect(await readdir(firstRemoteHome!)).toEqual(expect.arrayContaining(["sessions", "state_5.sqlite"]));
|
||||
expect(await readdir(firstRemoteHome!)).not.toContain("auth.json");
|
||||
const second = execute({ ...ctx, runId: "second", config: { ...ctx.config,
|
||||
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "generation-two" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
|
||||
runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } });
|
||||
if (fails) await expect(second).rejects.toThrow("fixture provider failed");
|
||||
else expect((await second).sessionId).toBe("ssh-thread");
|
||||
expect(calls).toBe(2);
|
||||
expect((await readdir(firstRemoteHome!)).sort()).toEqual(["sessions", "state_5.sqlite"]);
|
||||
expect((await stat(firstRemoteHome!)).mode & 0o077).toBe(0);
|
||||
if (!fails) {
|
||||
await writeFile(path.join(secondHome, "auth.json"), otherCredential);
|
||||
const switched = await execute({ ...ctx, runId: "other-account", config: { ...ctx.config,
|
||||
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
|
||||
runtime: { sessionId: first.sessionId!, sessionParams: first.sessionParams!, sessionDisplayId: first.sessionId!, taskKey: "task-one" } });
|
||||
expect(switched.sessionId).toBe("other-thread");
|
||||
expect(switched.sessionParams?.sshCodexAccountScope).not.toBe(first.sessionParams?.sshCodexAccountScope);
|
||||
expect(switched.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } });
|
||||
expect(await readdir(otherRemoteHome!)).not.toContain("auth.json");
|
||||
expect(await readFile(path.join(firstRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("fixture conversation");
|
||||
const oldShape = { sessionId: switched.sessionId!, cwd: workspace, remoteExecution: switched.sessionParams!.remoteExecution };
|
||||
const legacy = await execute({ ...ctx, runId: "old-saved-shape", config: { ...ctx.config,
|
||||
managedAiConnection: { ...ctx.config.managedAiConnection, identity: "other-account", sessionIdentity: "other-account" }, env: { CODEX_HOME: secondHome, OPENAI_API_KEY: "" } },
|
||||
runtime: { sessionId: switched.sessionId!, sessionParams: oldShape, sessionDisplayId: switched.sessionId!, taskKey: "task-one" } });
|
||||
expect(legacy.resultJson).toMatchObject({ sessionReset: { code: "codex_ssh_session_state_unavailable" } });
|
||||
expect(await readFile(path.join(otherRemoteHome!, "sessions", "rollout.jsonl"), "utf8")).toBe("other conversation");
|
||||
}
|
||||
for (const dir of privateRoots) await expect(stat(dir)).rejects.toThrow("ENOENT");
|
||||
} finally {
|
||||
runChildProcess.mockImplementation(originalRun);
|
||||
runSshCommand.mockReset().mockResolvedValue({ stdout: Buffer.from("{}").toString("base64"), stderr: "" });
|
||||
syncDirectoryToSsh.mockReset().mockResolvedValue(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("stages only the selected agent subscription and returns SSH refreshes to that account", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-selected-ssh-"));
|
||||
cleanupDirs.push(root);
|
||||
|
||||
@@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { inferOpenAiCompatibleBiller, resolveManagedOpenAiBilling, type AdapterExecutionContext, type AdapterExecutionResult } from "@paperclipai/adapter-utils";
|
||||
import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js";
|
||||
import { copyBackCodexAuth } from "./codex-auth-copyback.js";
|
||||
import { prepareCodexSshSessionState } from "./ssh-session-state.js";
|
||||
import {
|
||||
ensureCodexAuthCacheEntryDir,
|
||||
isCodexAuthCacheEnabled,
|
||||
@@ -783,6 +784,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// here so the outer `finally` can remove it on every exit path (teardown and
|
||||
// error), never only the happy path.
|
||||
let stagedCodexHomeDir: string | null = null;
|
||||
let sshSessionState: Awaited<ReturnType<typeof prepareCodexSshSessionState>> | null = null;
|
||||
let observabilityProfile: Awaited<ReturnType<typeof prepareCodexObservabilityProfile>> | null = null;
|
||||
let observabilityPreparationFailed = false;
|
||||
try {
|
||||
@@ -894,9 +896,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
// generic `restore` seam per asset before destroying the sandbox.
|
||||
// Target is the shared symlink SOURCE (what managed homes point
|
||||
// `auth.json` at), not the in-sandbox symlink.
|
||||
restore: async ({ assetDir, readFile }) =>
|
||||
void (await copyBackCodexAuth({
|
||||
readSandboxAuth: () => readFile(path.posix.join(assetDir, "auth.json")),
|
||||
restore: async ({ assetDir, readFile }) => {
|
||||
try { await copyBackCodexAuth({
|
||||
readSandboxAuth: () => readFile(path.posix.join(sshSessionState?.home ?? assetDir, "auth.json")),
|
||||
hostAuthPath: path.join(config.managedAiConnection ? effectiveCodexHome : resolveSharedCodexHomeDir(process.env), "auth.json"),
|
||||
log: (line) => onLog("stdout", `${line}\n`),
|
||||
// Additive cache write (sandbox to host): also cache the
|
||||
@@ -907,7 +909,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
resolveCacheEntryPath: config.managedAiConnection ? undefined : (accountId) =>
|
||||
ensureCodexAuthCacheEntryDir(process.env, accountId, agent.companyId),
|
||||
env: process.env,
|
||||
})),
|
||||
}); } finally { await sshSessionState?.cleanup(); }
|
||||
},
|
||||
// No `exclude` denylist: `stagedCodexHomeDir` already contains
|
||||
// ONLY the allowlisted files (auth/config/skills), so there is
|
||||
// nothing to filter out.
|
||||
@@ -928,10 +931,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
let paperclipBridge: Awaited<ReturnType<typeof startAdapterExecutionTargetPaperclipBridge>> = null;
|
||||
let executionError: unknown = null;
|
||||
try {
|
||||
const remoteCodexHome = executionTargetIsRemote && !useSshHostAuth
|
||||
let remoteCodexHome = executionTargetIsRemote && !useSshHostAuth
|
||||
? preparedExecutionTargetRuntime?.assetDirs.home ??
|
||||
path.posix.join(effectiveExecutionCwd, ".paperclip-runtime", "codex", "home")
|
||||
: null;
|
||||
if (executionTarget?.kind === "remote" && executionTarget.transport === "ssh" && executionTarget.spec.useHostAuth === false
|
||||
&& config.managedAiConnection && targetWorkspaceRealization?.mode === "in_place" && remoteCodexHome) {
|
||||
const account = parseObject(config.managedAiConnection);
|
||||
const identity = asString(account.sessionIdentity, asString(account.identity, ""));
|
||||
if (!identity) throw new Error("Managed SSH Codex execution requires an account identity");
|
||||
sshSessionState = await prepareCodexSshSessionState({
|
||||
remoteRoot: executionTarget.spec.remoteWorkspacePath, companyId: agent.companyId, agentId: agent.id,
|
||||
accountIdentity: identity, executionIdentity: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget),
|
||||
runId, saved: parseObject(runtime.sessionParams), stagedHome: remoteCodexHome,
|
||||
run: async command => {
|
||||
const result = await runAdapterExecutionTargetShellCommand(runId, runtimeExecutionTarget, command, { cwd: effectiveExecutionCwd, env: {}, timeoutSec: 15 });
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error("Cannot prepare or clean up SSH Codex conversation state");
|
||||
},
|
||||
});
|
||||
remoteCodexHome = sshSessionState.home;
|
||||
}
|
||||
await emitSandboxAuthPrecedenceWarningIfNeeded({
|
||||
runId,
|
||||
target: runtimeExecutionTarget,
|
||||
@@ -1138,6 +1157,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
const runtimeRemoteExecution = parseObject(runtimeSessionParams.remoteExecution);
|
||||
const canResumeSession =
|
||||
runtimeSessionId.length > 0 &&
|
||||
(!sshSessionState || sshSessionState.canResume) &&
|
||||
(runtimeSessionCwd.length === 0 || path.resolve(runtimeSessionCwd) === path.resolve(effectiveExecutionCwd)) &&
|
||||
adapterExecutionTargetSessionMatches(runtimeRemoteExecution, runtimeExecutionTarget);
|
||||
const codexTransientFallbackMode = readCodexTransientFallbackMode(context);
|
||||
@@ -1525,7 +1545,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
};
|
||||
}
|
||||
|
||||
const canFallbackToRuntimeSession = !isRetry && !forceFreshSession;
|
||||
const canFallbackToRuntimeSession = !isRetry && !forceFreshSession && (!sshSessionState || sshSessionState.canResume);
|
||||
const resolvedSessionId =
|
||||
attempt.parsed.sessionId ??
|
||||
(canFallbackToRuntimeSession ? (runtimeSessionId ?? runtime.sessionId ?? null) : null);
|
||||
@@ -1533,6 +1553,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
? ({
|
||||
sessionId: resolvedSessionId,
|
||||
cwd: effectiveExecutionCwd,
|
||||
...(sshSessionState ? { sshCodexConversationId: sshSessionState.conversationId, sshCodexAccountScope: sshSessionState.scope } : {}),
|
||||
...(executionTargetIsRemote
|
||||
? {
|
||||
remoteExecution: adapterExecutionTargetSessionIdentity(runtimeExecutionTarget),
|
||||
@@ -1636,6 +1657,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
resultJson: {
|
||||
stdout: attempt.proc.stdout,
|
||||
stderr: attempt.proc.stderr,
|
||||
...(sshSessionState && runtimeSessionId && !sshSessionState.canResume
|
||||
? { sessionReset: { code: "codex_ssh_session_state_unavailable" } } : {}),
|
||||
...(errorFamily ? { errorFamily } : {}),
|
||||
...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { shellQuote } from "@paperclipai/adapter-utils/ssh";
|
||||
|
||||
const STATE_ONLY = `for entry in "$home"/* "$home"/.[!.]* "$home"/..?*; do
|
||||
[ -e "$entry" ] || [ -L "$entry" ] || continue
|
||||
if [ ! -L "$entry" ]; then
|
||||
case "\${entry##*/}" in
|
||||
sessions|archived_sessions|session_index.jsonl|history.jsonl|state_*.sqlite|state_*.sqlite-*|thread_history_*.sqlite|thread_history_*.sqlite-*) continue ;;
|
||||
esac
|
||||
fi
|
||||
rm -rf -- "$entry"
|
||||
done`;
|
||||
|
||||
export async function prepareCodexSshSessionState(input: {
|
||||
remoteRoot: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
accountIdentity: string;
|
||||
executionIdentity: unknown;
|
||||
runId: string;
|
||||
saved: Record<string, unknown>;
|
||||
stagedHome: string;
|
||||
run: (command: string) => Promise<void>;
|
||||
}) {
|
||||
const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
const scope = hash([input.companyId, input.agentId, input.accountIdentity, input.executionIdentity]);
|
||||
const savedId = input.saved.sshCodexConversationId;
|
||||
const canResume = input.saved.sshCodexAccountScope === scope && typeof savedId === "string" && /^[a-f0-9]{64}$/.test(savedId);
|
||||
const conversationId = canResume ? savedId : hash(input.runId);
|
||||
const home = path.posix.join(input.remoteRoot, ".paperclip-runtime", "codex", "conversations", scope, conversationId);
|
||||
const prefix = `set -eu\numask 077\nhome=${shellQuote(home)}\n[ ! -L "$home" ]\n`;
|
||||
const cleanup = () => input.run(`${prefix}if [ -d "$home" ]; then\n${STATE_ONLY}\nfi`);
|
||||
try {
|
||||
await input.run(`${prefix}mkdir -p "$home"\nchmod 700 "$home"\n${STATE_ONLY}\ncp -RP ${shellQuote(input.stagedHome + "/.")} "$home/"`);
|
||||
} catch (error) {
|
||||
await cleanup();
|
||||
throw error;
|
||||
}
|
||||
return { home, scope, conversationId, canResume, cleanup };
|
||||
}
|
||||
@@ -64,7 +64,10 @@ vi.mock("@paperclipai/adapter-utils/execution-target", () => ({
|
||||
prepareAdapterExecutionTargetRuntime: (...args: unknown[]) =>
|
||||
(mocks.prepareRuntimeMock as (...args: unknown[]) => unknown)(...args),
|
||||
readAdapterExecutionTarget: (input: { executionTarget?: unknown }) => input.executionTarget ??
|
||||
(mocks.state.isRemote ? { kind: "remote", transport: "ssh" } : { kind: "local" }),
|
||||
(mocks.state.isRemote ? { kind: "remote", transport: "ssh", remoteCwd: "/remote/workspace", spec: {
|
||||
host: "fixture.invalid", port: 22, username: "fixture", remoteCwd: "/remote/workspace", remoteWorkspacePath: "/remote/workspace",
|
||||
privateKey: null, knownHosts: null, strictHostKeyChecking: true,
|
||||
} } : { kind: "local" }),
|
||||
resolveAdapterExecutionTargetCommandForLogs: (...args: unknown[]) =>
|
||||
(mocks.resolveCommandForLogsMock as (...args: unknown[]) => unknown)(...args),
|
||||
resolveAdapterExecutionTargetTimeoutSec: (_target: unknown, timeoutSec: number) => timeoutSec,
|
||||
|
||||
@@ -27,6 +27,8 @@ import { legacyExecutionNeedsReconciliation } from "../services/legacy-execution
|
||||
import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js";
|
||||
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
|
||||
import { dotRunnerBroker } from "../services/dot-runner-broker.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import * as executionConfiguration from "../services/heartbeat/execution-configuration.js";
|
||||
|
||||
const captureRunFailure = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../sentry.js", async (importOriginal) => ({
|
||||
@@ -446,6 +448,52 @@ describe("durable, authorized connection routing", () => {
|
||||
expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: String(selected.runtimeConfig.model), taskSessionParams: { __paperclipConfiguredModel: selected.runtimeConfig.model, __paperclipConfigFingerprint: previous.fingerprint, __paperclipConfigFingerprintVersion: previous.version, __paperclipConfigCategoryFingerprints: previous.categoryFingerprints }, configMetadata: current, compatibleConfigMetadata }).reset).toBe(false);
|
||||
} finally { await before.cleanup(); await after.cleanup(); }
|
||||
});
|
||||
it.each([
|
||||
{ useHostAuth: undefined, nativeHomeMode: undefined }, { useHostAuth: true, nativeHomeMode: undefined },
|
||||
{ useHostAuth: false, nativeHomeMode: undefined }, { useHostAuth: false, nativeHomeMode: "preconfigured" },
|
||||
])("applies SSH authentication before selecting a busy pool (%j)", async ({ useHostAuth, nativeHomeMode }) => {
|
||||
const original = await instanceSettingsService(db).getExperimental();
|
||||
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, enableNativeRunner: true, enableEnvironments: true, enableWorktreeRunExecution: true });
|
||||
const pool = await makePool({ members: [members[0]!] });
|
||||
const environment = await environmentService(db).create(companyId, { name: `SSH routing fixture ${useHostAuth} ${nativeHomeMode}`, driver: "ssh", status: "active", envVars: {}, config: {
|
||||
host: "ssh.example.test", username: "fixture", remoteWorkspacePath: "/workspace", ...(useHostAuth === undefined ? {} : { useHostAuth }),
|
||||
} });
|
||||
const configured = { provider: "acpx", acpxAgent: "claude", model: "fixture-host-model", ...(nativeHomeMode ? { homeMode: nativeHomeMode } : {}) };
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "SSH routing fixture", status: "idle", adapterType: "paperclip_runner",
|
||||
adapterConfig: configured, defaultEnvironmentId: environment.id,
|
||||
runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id }, heartbeat: { enabled: false, wakeOnDemand: true } } }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "SSH pool routing fixture", status: "todo", assigneeAgentId: agent.id, responsibleUserId: "alice", createdByUserId: "alice" }).returning();
|
||||
proposal.mockClear();
|
||||
proposal.mockImplementation(async () => ({ kind: "exhausted", retryAt: new Date(Date.now() + 60_000).toISOString(), skipped: {} }));
|
||||
const preparation = vi.spyOn(executionConfiguration, "prepareHeartbeatExecutionConfiguration").mockRejectedValue(new Error("fixture stopped before provider configuration"));
|
||||
const heartbeat = heartbeatService(db, { pluginWorkerManager: worker, runtimeEnv: {} });
|
||||
try {
|
||||
const run = await heartbeat.wakeup(agent.id, { source: "assignment", reason: "issue_assigned", triggerDetail: "system",
|
||||
payload: { issueId: issue.id }, contextSnapshot: { issueId: issue.id, responsibleUserId: "alice" } });
|
||||
expect(run).not.toBeNull();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const saved = await heartbeat.getRun(run!.id);
|
||||
if (useHostAuth === false) {
|
||||
expect(proposal).toHaveBeenCalled();
|
||||
expect(preparation).not.toHaveBeenCalled();
|
||||
expect(saved?.errorCode).toBe("ai_connection_pool_exhausted");
|
||||
} else {
|
||||
expect(proposal).not.toHaveBeenCalled();
|
||||
expect(saved?.contextSnapshot?.aiRouterSelection).toBeUndefined();
|
||||
expect(saved?.error).toContain("fixture stopped before provider configuration");
|
||||
expect(preparation).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ config: expect.objectContaining({
|
||||
aiBinding: undefined, sourceWorkspaceConfig: expect.objectContaining(configured),
|
||||
}) }));
|
||||
}
|
||||
} finally {
|
||||
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agent.id));
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
preparation.mockRestore();
|
||||
proposal.mockImplementation(defaultProposal);
|
||||
await instanceSettingsService(db).updateExperimental(original);
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("admits a healthy pool member when the saved Grok connection is held", async () => {
|
||||
const original = await instanceSettingsService(db).getExperimental();
|
||||
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true,
|
||||
|
||||
@@ -25,7 +25,7 @@ import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import * as sentry from "../sentry.js";
|
||||
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
import { selectHeartbeatEnvironment, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js";
|
||||
import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { executionWorkspaceService } from "../services/execution-workspaces.js";
|
||||
import { buildExecutionWorkspaceAdapterConfig } from "../services/execution-workspace-policy.js";
|
||||
@@ -41,6 +41,21 @@ function deferred<T>() {
|
||||
const skillFailures = vi.hoisted(() => new Map<string, string>());
|
||||
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
|
||||
|
||||
describe("SSH authentication before account routing", () => {
|
||||
it.each([
|
||||
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "agent", expected: "host" },
|
||||
{ agentDefaultEnvironmentId: null, lowTrustIssueEnvironmentId: null, executionMode: "any", expectedId: "instance", expected: "host" },
|
||||
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: "sandbox", executionMode: "any", expectedId: "sandbox", expected: null },
|
||||
{ agentDefaultEnvironmentId: "agent", lowTrustIssueEnvironmentId: null, executionMode: "kubernetes", expectedId: null, expected: null },
|
||||
] as const)("honors environment precedence and forced execution policy: %j", async value => {
|
||||
const get = vi.fn(async () => ({ defaultEnvironmentId: "instance", general: { executionMode: value.executionMode } }));
|
||||
const getById = vi.fn(async (id: string) => ({ id, driver: id === "sandbox" ? "sandbox" : "ssh", config: {} } as Environment));
|
||||
expect(await resolveSshAuthenticationBeforeRouting({ ...value, instanceSettings: { get }, environmentsSvc: { getById } })).toBe(value.expected);
|
||||
if (value.expectedId) expect(getById).toHaveBeenCalledExactlyOnceWith(value.expectedId);
|
||||
else expect(getById).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Runner home mode during recovery preparation", () => {
|
||||
const fingerprint = (effectiveAdapterConfig: Record<string, unknown>) => buildEffectiveRunSessionConfigMetadata({
|
||||
adapterType: "paperclip_runner", effectiveAdapterConfig, agentRuntimeConfig: {}, issueOverrides: null,
|
||||
|
||||
@@ -24,7 +24,7 @@ vi.mock("node:child_process", async importOriginal => {
|
||||
spawn: (file: string, args: string[], options: Parameters<typeof actual.spawn>[2]) => {
|
||||
if (file !== "ssh") return actual.spawn(file, args, options);
|
||||
mocks.wire.push({ file, args, stdin: true });
|
||||
return actual.spawn(process.execPath, ["-e", 'process.stdout.write(require("node:fs").readFileSync(0,"utf8"))'], options);
|
||||
return actual.spawn(process.execPath, ["-e", 'const wire = require("node:fs").readFileSync(0); process.stdout.write(wire.subarray(wire.indexOf(10) + 1))'], options);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -3391,7 +3391,7 @@ export function agentRoutes(
|
||||
});
|
||||
type = selection.adapterType;
|
||||
adapterConfigForTest = { ...selection.adapterConfig,
|
||||
...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" && requestedEnvironment.config.useHostAuth !== false ? { homeMode: "preconfigured" } : {}) };
|
||||
...(type === "paperclip_runner" && requestedEnvironment?.driver === "ssh" ? { homeMode: requestedEnvironment.config.useHostAuth === false ? "isolated" : "preconfigured" } : {}) };
|
||||
if (requestedEnvironment) await assertAdapterTestEnvironmentForCompany(companyId, requestedEnvironment.id, type);
|
||||
const adapter = requireServerAdapter(type);
|
||||
if (!retainsSavedExecution) await assertSelectableAdapterType(type, adapterConfigForTest, companyId);
|
||||
|
||||
@@ -58,7 +58,7 @@ import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.
|
||||
import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability";
|
||||
import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js";
|
||||
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
|
||||
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
|
||||
import { selectHeartbeatEnvironment, resolveSshAuthenticationBeforeRouting } from "./heartbeat/environment-selection.js";
|
||||
import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js";
|
||||
import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
|
||||
import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js";
|
||||
@@ -2666,8 +2666,17 @@ export function heartbeatService(
|
||||
if ("homeMode" in homeConfig) config = { ...config, homeMode: homeConfig.homeMode };
|
||||
}
|
||||
const persistedRouterPoolId = routerHasPersistedInput ? readNonEmptyString(parseObject(context.aiRouterSelection).poolId) : null;
|
||||
const sshAuthentication = routerHasPersistedInput ? null : await resolveSshAuthenticationBeforeRouting({
|
||||
agentDefaultEnvironmentId: agent.defaultEnvironmentId,
|
||||
lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review"
|
||||
? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, { includeEnvironmentId: true })?.environmentId ?? null : null,
|
||||
environmentsSvc, instanceSettings,
|
||||
});
|
||||
if (agent.adapterType === "paperclip_runner" && sshAuthentication) {
|
||||
config = { ...config, homeMode: sshAuthentication === "host" ? "preconfigured" : "isolated" };
|
||||
}
|
||||
const preconfiguredHome = agent.adapterType === "paperclip_runner" && config.homeMode === "preconfigured";
|
||||
const requestedAiBinding = preconfiguredHome ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
|
||||
const requestedAiBinding = preconfiguredHome || sshAuthentication === "host" ? undefined : persistedRouterPoolId ? { mode: "router" as const, connectionId: persistedRouterPoolId } : agent.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(agent.runtimeConfig.aiConnection) : undefined;
|
||||
let aiBinding = requestedAiBinding?.mode === "router" ? undefined : requestedAiBinding;
|
||||
const originalAiIssueOverrides = issueAssigneeOverrides;
|
||||
if (requestedAiBinding?.mode === "router") {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./e
|
||||
import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { readGitCacheExpiration } from "../git-workspace-restoration.js";
|
||||
import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared";
|
||||
import type { ExecutionWorkspace, InstanceSettings, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
@@ -39,6 +39,23 @@ import { resolveNativeRecoveryExecutionWorkspaceBinding, resolveExecutionWorkspa
|
||||
|
||||
type Issue = typeof issues.$inferSelect;
|
||||
|
||||
export async function resolveSshAuthenticationBeforeRouting(input: {
|
||||
agentDefaultEnvironmentId: string | null;
|
||||
lowTrustIssueEnvironmentId: string | null;
|
||||
environmentsSvc: Pick<ReturnType<typeof environmentService>, "getById">;
|
||||
instanceSettings: { get(): Promise<{
|
||||
defaultEnvironmentId: InstanceSettings["defaultEnvironmentId"];
|
||||
general: Pick<InstanceSettings["general"], "executionMode">;
|
||||
}> };
|
||||
}): Promise<"host" | "managed" | null> {
|
||||
const settings = await input.instanceSettings.get();
|
||||
if (isExecutionForcedToKubernetes({ executionMode: settings.general.executionMode })) return null;
|
||||
const id = input.lowTrustIssueEnvironmentId ?? input.agentDefaultEnvironmentId ?? settings.defaultEnvironmentId;
|
||||
if (!id) return null;
|
||||
const environment = await input.environmentsSvc.getById(id);
|
||||
return environment?.driver === "ssh" ? environment.config.useHostAuth === false ? "managed" : "host" : null;
|
||||
}
|
||||
|
||||
/** Selects policy and reuse inputs; lease acquisition and cleanup belong to the executor. */
|
||||
export interface HeartbeatEnvironmentSelectionInput {
|
||||
run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
|
||||
@@ -102,7 +102,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
const useSshHostAuth = selectedEnvironmentForConfig?.driver === "ssh" && selectedEnvironmentForConfig.config.useHostAuth !== false && !persistedNativeExecutionInput;
|
||||
const sourceWorkspaceConfig = agent.adapterType === "paperclip_runner"
|
||||
? { ...input.config.sourceWorkspaceConfig,
|
||||
homeMode: bindHeartbeatRunnerHomeMode({ ...parseObject(agent.adapterConfig), ...(useSshHostAuth ? { homeMode: "preconfigured" } : {}) }, persistedNativeExecutionInput).homeMode }
|
||||
homeMode: bindHeartbeatRunnerHomeMode({ ...parseObject(agent.adapterConfig), ...(selectedEnvironmentForConfig?.driver === "ssh" && !persistedNativeExecutionInput ? { homeMode: useSshHostAuth ? "preconfigured" : "isolated" } : {}) }, persistedNativeExecutionInput).homeMode }
|
||||
: input.config.sourceWorkspaceConfig;
|
||||
const preconfiguredHome = agent.adapterType === "paperclip_runner" && sourceWorkspaceConfig.homeMode === "preconfigured";
|
||||
if (preconfiguredHome && pinnedSkillTest) {
|
||||
|
||||
@@ -58,7 +58,7 @@ describe("native SSH deliverable output budget", () => {
|
||||
});
|
||||
|
||||
it("still delivers every byte when SSH consumes the upload", async () => {
|
||||
await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst hash = require('node:crypto').createHash('sha256');\nprocess.stdin.on('data', chunk => hash.update(chunk));\nprocess.stdin.on('end', () => process.stdout.write(hash.digest('hex')));\n`);
|
||||
await writeFile(join(root, "ssh"), `#!${process.execPath}\nconst chunks = [];\nprocess.stdin.on('data', chunk => chunks.push(chunk));\nprocess.stdin.on('end', () => { const wire = Buffer.concat(chunks); const end = wire.indexOf(10); if (end < 0) process.exit(1); process.stdout.write(require('node:crypto').createHash('sha256').update(wire.subarray(end + 1)).digest('hex')); });\n`);
|
||||
const stdin = "synthetic upload bytes\0".repeat(32_768);
|
||||
const result = await createRunner().execute({ command: "node", args: ["unused"], stdin, timeoutMs: 5000 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
|
||||
@@ -392,7 +392,7 @@
|
||||
"packages/shared/src/types/decision.ts": "b0d40077d481ce6180de6b02bad6df5bb0fbf977",
|
||||
"packages/shared/src/types/document-annotation.ts": "7769a42f1ccd707a2259774e7fe6aa536b5d183f",
|
||||
"packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc",
|
||||
"packages/shared/src/types/environment.ts": "a380b9ce9d2d1763011e9adaa58615a3ef1e56c5",
|
||||
"packages/shared/src/types/environment.ts": "16b9d9598abfa2583602181b44483b0fa24d5bc2",
|
||||
"packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b",
|
||||
"packages/shared/src/types/execution-projection.ts": "e72e641602b296f6d3e16d07700629df3039a7b1",
|
||||
"packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1",
|
||||
|
||||
Reference in new issue
Block a user