Files
PaperClipAI/packages/adapter-utils/src/remote-managed-runtime.ts
T
Devin FoleyandPaperclip 0631cf1ec3 Distinguish missing SSH credential files from read failures
Routed Codex accounts do not create auth.json. Report a confirmed missing file as ENOENT so the existing guarded credential copyback can skip it. Require the remote marker and exit status together, and preserve connection and other read errors.

Reproduced the failure with the real shell command. Verified 325 tests across the SSH runtime, Codex probes and credential merge, Codex remote execution, and ACP executor. Adapter-utils and Codex typechecks passed. The routed probe retains its config-only home and still removes temporary staging files.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-11-12 19:47:14 -08:00

317 lines
12 KiB
TypeScript

import path from "node:path";
import { GIT_ARCHIVE_EXCLUDES, PROJECT_REPOSITORIES_DIR } from "./git-workspace-sync.js";
import {
type SshRemoteExecutionSpec,
prepareWorkspaceForSshExecution,
runSshCommand,
restoreWorkspaceFromSshExecution,
syncDirectoryToSsh,
} from "./ssh.js";
import {
mergeExcludes,
referencedSourceIgnoreExcludeEntries,
type SandboxAdditionalSource,
type SandboxManagedRuntimeAssetRestoreContext,
} from "./sandbox-managed-runtime.js";
import { captureDirectorySnapshot, type DirectorySnapshot } from "./workspace-restore-merge.js";
import type { RuntimeProgressSink } from "./runtime-progress.js";
// The fixed heavy-directory excludes every referenced project drops,
// regardless of its ignore resolution. A `git`-resolved project additionally
// drops its own resolved ignored paths (see `referencedSourceIgnoreExcludeEntries`
// and the per-project merge below); an `other` project keeps only this set.
const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [
"node_modules",
"vendor",
"dist",
"build",
"out",
"coverage",
".next",
".turbo",
".cache",
".git",
].flatMap((entry) => [entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`]);
export interface RemoteManagedRuntimeAsset {
key: string;
localDir: string;
followSymlinks?: boolean;
exclude?: string[];
restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise<void>;
}
export interface PreparedRemoteManagedRuntime {
spec: SshRemoteExecutionSpec;
workspaceLocalDir: string;
workspaceRemoteDir: string;
runtimeRootDir: string;
assetDirs: Record<string, string>;
/**
* Remote directory of each additional (referenced) project that staged
* successfully, keyed by `projectId`. A project whose staging failed is
* absent (per-project failure isolation).
*/
additionalSourceDirs: Record<string, string>;
disposeRuntime(): Promise<void>;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
function asObject(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function asString(value: unknown): string {
return typeof value === "string" ? value : "";
}
function asNumber(value: unknown): number {
return typeof value === "number" ? value : Number(value);
}
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'"'"'`)}'`;
}
async function readRemoteFile(spec: SshRemoteExecutionSpec, remotePath: string): Promise<Buffer> {
const missingMarker = "paperclip:remote-file-missing";
const missingExitCode = 44;
const file = shellQuote(remotePath);
const parent = shellQuote(path.posix.dirname(remotePath));
/* Confirm the parent is searchable so a permission error cannot masquerade as absence. */
const command = `if [ -d ${parent} ] && [ -x ${parent} ] && [ ! -e ${file} ]; then printf '%s\\n' '${missingMarker}' >&2; exit ${missingExitCode}; fi; base64 < ${file}`;
try {
const result = await runSshCommand(spec, command, { maxBuffer: 1024 * 1024 });
return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64");
} catch (error) {
const failure = error as { code?: unknown; stderr?: unknown } | null;
if (failure?.code === missingExitCode && typeof failure.stderr === "string" && failure.stderr.trim() === missingMarker) {
throw Object.assign(new Error("Remote runtime file does not exist"), { code: "ENOENT" });
}
throw error;
}
}
export function buildRemoteExecutionSessionIdentity(spec: SshRemoteExecutionSpec | null) {
if (!spec) return null;
return {
transport: "ssh",
...(typeof spec.useHostAuth === "boolean" ? { useHostAuth: spec.useHostAuth } : {}),
host: spec.host,
port: spec.port,
username: spec.username,
remoteCwd: spec.remoteCwd,
} as const;
}
export function remoteExecutionSessionMatches(saved: unknown, current: SshRemoteExecutionSpec | null): boolean {
const currentIdentity = buildRemoteExecutionSessionIdentity(current);
if (!currentIdentity) return false;
const parsedSaved = asObject(saved);
return (
asString(parsedSaved.transport) === currentIdentity.transport &&
parsedSaved.useHostAuth === currentIdentity.useHostAuth &&
asString(parsedSaved.host) === currentIdentity.host &&
asNumber(parsedSaved.port) === currentIdentity.port &&
asString(parsedSaved.username) === currentIdentity.username &&
asString(parsedSaved.remoteCwd) === currentIdentity.remoteCwd
);
}
export async function prepareRemoteManagedRuntime(input: {
spec: SshRemoteExecutionSpec;
runId: string;
adapterKey: string;
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
privateRuntime?: boolean;
retainRuntime?: boolean;
workspaceFileMode?: "all";
workspaceExclude?: string[];
assets?: RemoteManagedRuntimeAsset[];
/** Referenced (additional) projects to stage as plain, read-only trees. */
additionalSources?: SandboxAdditionalSource[];
// Upload progress sink. Threaded for the byte-counting transport rewrite; the
// child task wires it into the workspace/asset transfers.
onProgress?: RuntimeProgressSink;
}): Promise<PreparedRemoteManagedRuntime> {
const baseWorkspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
const syncWorkspace = input.syncWorkspace !== false;
const workspaceRemoteDir = syncWorkspace
? path.posix.join(
baseWorkspaceRemoteDir,
".paperclip-runtime",
"runs",
input.runId,
"workspace",
)
: baseWorkspaceRemoteDir;
let runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
let privateRuntimeDir: string | null = null;
const cleanupPrivateRuntime = async () => {
if (!privateRuntimeDir) return;
await runSshCommand(input.spec, `rm -rf -- ${shellQuote(privateRuntimeDir)}`, { timeoutMs: 15_000 });
privateRuntimeDir = null;
};
const preparedWorkspace = syncWorkspace
? await prepareWorkspaceForSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
onProgress: input.onProgress,
workspaceFileMode: input.workspaceFileMode,
workspaceExclude: input.workspaceExclude,
})
: null;
const projectRepositories = preparedWorkspace?.repositories ?? [];
const baselineSnapshot = preparedWorkspace
? await captureDirectorySnapshot(input.workspaceLocalDir, {
exclude: preparedWorkspace.gitBacked
? [
...GIT_ARCHIVE_EXCLUDES,
".paperclip-runtime",
...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : preparedWorkspace.excludedPaths ?? []),
]
: [".paperclip-runtime", ...(preparedWorkspace.excludedPaths ?? []), ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])],
})
: null;
const repositoryBaselines: Array<{ path: string; baselineSnapshot: DirectorySnapshot }> = [];
for (const repository of projectRepositories) {
repositoryBaselines.push({
path: repository,
baselineSnapshot: await captureDirectorySnapshot(path.join(input.workspaceLocalDir, repository), {
exclude: [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"],
}),
});
}
const assetDirs: Record<string, string> = {};
try {
if (input.privateRuntime) {
const result = await runSshCommand(input.spec, "umask 077; mktemp -d /tmp/paperclip-ssh-runtime.XXXXXXXXXX", { timeoutMs: 15_000 });
const directory = result.stdout.trim();
if (!/^\/tmp\/paperclip-ssh-runtime\.[A-Za-z0-9]+$/.test(directory)) throw new Error("SSH private runtime directory unavailable");
runtimeRootDir = privateRuntimeDir = directory;
}
for (const asset of input.assets ?? []) {
if (input.privateRuntime && !/^[A-Za-z0-9_-]+$/.test(asset.key)) throw new Error("Invalid SSH runtime asset key");
const remoteDir = path.posix.join(runtimeRootDir, asset.key);
assetDirs[asset.key] = remoteDir;
await syncDirectoryToSsh({
spec: input.spec,
localDir: asset.localDir,
remoteDir,
followSymlinks: asset.followSymlinks,
exclude: asset.exclude,
onProgress: input.onProgress,
progressLabel: asset.key,
});
}
} catch (error) {
try {
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
baselineSnapshot,
restoreGitHistory: preparedWorkspace.gitBacked,
onProgress: input.onProgress,
repositories: repositoryBaselines,
});
} finally { await cleanupPrivateRuntime(); }
throw error;
}
// Stage each referenced (additional) project as a plain, read-only tree in its
// OWN isolated remote directory (`project-<projectId>`). Additional sources
// never get the anchor's git-history/overlay semantics. Per-project failure
// isolation: one project's failure logs a warning and is skipped; the run and
// the other projects continue (no workspace restore, unlike an asset failure).
const additionalSourceDirs: Record<string, string> = {};
for (const source of input.additionalSources ?? []) {
const { localPath, projectId, ignoreResolution } = source;
try {
if (!path.posix.isAbsolute(localPath)) {
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
}
if (
projectId.length === 0 ||
projectId.includes("/") ||
projectId.includes("\\") ||
projectId.includes("..")
) {
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
}
// Fail closed: a project whose ignore resolution failed is not staged at
// all — the existing per-project skip-and-warn path below handles it.
if (ignoreResolution.kind === "failed") {
throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`);
}
const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`);
const exclude = mergeExcludes(
REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
referencedSourceIgnoreExcludeEntries(ignoreResolution),
);
await syncDirectoryToSsh({
spec: input.spec,
localDir: localPath,
remoteDir,
exclude,
onProgress: input.onProgress,
progressLabel: `project-${projectId}`,
});
additionalSourceDirs[projectId] = remoteDir;
} catch (error) {
console.warn(
`[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,
);
}
}
const restore = async (onProgress?: RuntimeProgressSink) => {
let failure: unknown;
try {
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
baselineSnapshot,
restoreGitHistory: preparedWorkspace.gitBacked,
onProgress,
repositories: repositoryBaselines,
});
} catch (error) { failure = error; }
for (const asset of input.assets ?? []) {
if (!asset.restore) continue;
try { await asset.restore({
assetDir: path.posix.join(runtimeRootDir, asset.key),
readFile: (remotePath) => readRemoteFile(input.spec, remotePath),
}); } catch (error) { failure ??= error; }
}
if (!input.retainRuntime) {
try { await cleanupPrivateRuntime(); } catch (error) { failure ??= error; }
}
if (failure) throw failure;
};
let privateRestore: Promise<void> | undefined;
return {
spec: input.spec,
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
runtimeRootDir,
assetDirs,
additionalSourceDirs,
disposeRuntime: cleanupPrivateRuntime,
restoreWorkspace: (onProgress?: RuntimeProgressSink) => {
if (!input.privateRuntime || input.retainRuntime) return restore(onProgress);
return privateRestore ??= restore(onProgress);
},
};
}