Files
PaperClipAI/packages/adapter-utils/src/remote-managed-runtime.ts
T
Devin FoleyandPaperclip c3fbe23431 Keep SSH file-read status separate from connection warnings
Send the missing-file marker on stdout so SSH host-key warnings on stderr do not invalidate a confirmed absence. Continue requiring the matching numeric exit status, and preserve all unrelated errors.

Added a failing real-shell warning regression, then verified 60 affected runtime, credential, probe, and execution tests. Adapter-utils and Codex typechecks passed.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-11-12 19:49:10 -08:00

317 lines
12 KiB
TypeScript

import path from "node:path";
import { GIT_ARCHIVE_EXCLUDES, PROJECT_REPOSITORIES_DIR } from "./git-workspace-sync.js";
import {
type SshRemoteExecutionSpec,
prepareWorkspaceForSshExecution,
runSshCommand,
restoreWorkspaceFromSshExecution,
syncDirectoryToSsh,
} from "./ssh.js";
import {
mergeExcludes,
referencedSourceIgnoreExcludeEntries,
type SandboxAdditionalSource,
type SandboxManagedRuntimeAssetRestoreContext,
} from "./sandbox-managed-runtime.js";
import { captureDirectorySnapshot, type DirectorySnapshot } from "./workspace-restore-merge.js";
import type { RuntimeProgressSink } from "./runtime-progress.js";
// The fixed heavy-directory excludes every referenced project drops,
// regardless of its ignore resolution. A `git`-resolved project additionally
// drops its own resolved ignored paths (see `referencedSourceIgnoreExcludeEntries`
// and the per-project merge below); an `other` project keeps only this set.
const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [
"node_modules",
"vendor",
"dist",
"build",
"out",
"coverage",
".next",
".turbo",
".cache",
".git",
].flatMap((entry) => [entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`]);
export interface RemoteManagedRuntimeAsset {
key: string;
localDir: string;
followSymlinks?: boolean;
exclude?: string[];
restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise<void>;
}
export interface PreparedRemoteManagedRuntime {
spec: SshRemoteExecutionSpec;
workspaceLocalDir: string;
workspaceRemoteDir: string;
runtimeRootDir: string;
assetDirs: Record<string, string>;
/**
* Remote directory of each additional (referenced) project that staged
* successfully, keyed by `projectId`. A project whose staging failed is
* absent (per-project failure isolation).
*/
additionalSourceDirs: Record<string, string>;
disposeRuntime(): Promise<void>;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
function asObject(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function asString(value: unknown): string {
return typeof value === "string" ? value : "";
}
function asNumber(value: unknown): number {
return typeof value === "number" ? value : Number(value);
}
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'"'"'`)}'`;
}
async function readRemoteFile(spec: SshRemoteExecutionSpec, remotePath: string): Promise<Buffer> {
const missingMarker = "paperclip:remote-file-missing";
const missingExitCode = 44;
const file = shellQuote(remotePath);
const parent = shellQuote(path.posix.dirname(remotePath));
/* Confirm the parent is searchable so a permission error cannot masquerade as absence. */
const command = `if [ -d ${parent} ] && [ -x ${parent} ] && [ ! -e ${file} ]; then printf '%s\\n' '${missingMarker}'; exit ${missingExitCode}; fi; base64 < ${file}`;
try {
const result = await runSshCommand(spec, command, { maxBuffer: 1024 * 1024 });
return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64");
} catch (error) {
const failure = error as { code?: unknown; stdout?: unknown } | null;
if (failure?.code === missingExitCode && typeof failure.stdout === "string" && failure.stdout.trim() === missingMarker) {
throw Object.assign(new Error("Remote runtime file does not exist"), { code: "ENOENT" });
}
throw error;
}
}
export function buildRemoteExecutionSessionIdentity(spec: SshRemoteExecutionSpec | null) {
if (!spec) return null;
return {
transport: "ssh",
...(typeof spec.useHostAuth === "boolean" ? { useHostAuth: spec.useHostAuth } : {}),
host: spec.host,
port: spec.port,
username: spec.username,
remoteCwd: spec.remoteCwd,
} as const;
}
export function remoteExecutionSessionMatches(saved: unknown, current: SshRemoteExecutionSpec | null): boolean {
const currentIdentity = buildRemoteExecutionSessionIdentity(current);
if (!currentIdentity) return false;
const parsedSaved = asObject(saved);
return (
asString(parsedSaved.transport) === currentIdentity.transport &&
parsedSaved.useHostAuth === currentIdentity.useHostAuth &&
asString(parsedSaved.host) === currentIdentity.host &&
asNumber(parsedSaved.port) === currentIdentity.port &&
asString(parsedSaved.username) === currentIdentity.username &&
asString(parsedSaved.remoteCwd) === currentIdentity.remoteCwd
);
}
export async function prepareRemoteManagedRuntime(input: {
spec: SshRemoteExecutionSpec;
runId: string;
adapterKey: string;
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
privateRuntime?: boolean;
retainRuntime?: boolean;
workspaceFileMode?: "all";
workspaceExclude?: string[];
assets?: RemoteManagedRuntimeAsset[];
/** Referenced (additional) projects to stage as plain, read-only trees. */
additionalSources?: SandboxAdditionalSource[];
// Upload progress sink. Threaded for the byte-counting transport rewrite; the
// child task wires it into the workspace/asset transfers.
onProgress?: RuntimeProgressSink;
}): Promise<PreparedRemoteManagedRuntime> {
const baseWorkspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
const syncWorkspace = input.syncWorkspace !== false;
const workspaceRemoteDir = syncWorkspace
? path.posix.join(
baseWorkspaceRemoteDir,
".paperclip-runtime",
"runs",
input.runId,
"workspace",
)
: baseWorkspaceRemoteDir;
let runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
let privateRuntimeDir: string | null = null;
const cleanupPrivateRuntime = async () => {
if (!privateRuntimeDir) return;
await runSshCommand(input.spec, `rm -rf -- ${shellQuote(privateRuntimeDir)}`, { timeoutMs: 15_000 });
privateRuntimeDir = null;
};
const preparedWorkspace = syncWorkspace
? await prepareWorkspaceForSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
onProgress: input.onProgress,
workspaceFileMode: input.workspaceFileMode,
workspaceExclude: input.workspaceExclude,
})
: null;
const projectRepositories = preparedWorkspace?.repositories ?? [];
const baselineSnapshot = preparedWorkspace
? await captureDirectorySnapshot(input.workspaceLocalDir, {
exclude: preparedWorkspace.gitBacked
? [
...GIT_ARCHIVE_EXCLUDES,
".paperclip-runtime",
...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : preparedWorkspace.excludedPaths ?? []),
]
: [".paperclip-runtime", ...(preparedWorkspace.excludedPaths ?? []), ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])],
})
: null;
const repositoryBaselines: Array<{ path: string; baselineSnapshot: DirectorySnapshot }> = [];
for (const repository of projectRepositories) {
repositoryBaselines.push({
path: repository,
baselineSnapshot: await captureDirectorySnapshot(path.join(input.workspaceLocalDir, repository), {
exclude: [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"],
}),
});
}
const assetDirs: Record<string, string> = {};
try {
if (input.privateRuntime) {
const result = await runSshCommand(input.spec, "umask 077; mktemp -d /tmp/paperclip-ssh-runtime.XXXXXXXXXX", { timeoutMs: 15_000 });
const directory = result.stdout.trim();
if (!/^\/tmp\/paperclip-ssh-runtime\.[A-Za-z0-9]+$/.test(directory)) throw new Error("SSH private runtime directory unavailable");
runtimeRootDir = privateRuntimeDir = directory;
}
for (const asset of input.assets ?? []) {
if (input.privateRuntime && !/^[A-Za-z0-9_-]+$/.test(asset.key)) throw new Error("Invalid SSH runtime asset key");
const remoteDir = path.posix.join(runtimeRootDir, asset.key);
assetDirs[asset.key] = remoteDir;
await syncDirectoryToSsh({
spec: input.spec,
localDir: asset.localDir,
remoteDir,
followSymlinks: asset.followSymlinks,
exclude: asset.exclude,
onProgress: input.onProgress,
progressLabel: asset.key,
});
}
} catch (error) {
try {
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
baselineSnapshot,
restoreGitHistory: preparedWorkspace.gitBacked,
onProgress: input.onProgress,
repositories: repositoryBaselines,
});
} finally { await cleanupPrivateRuntime(); }
throw error;
}
// Stage each referenced (additional) project as a plain, read-only tree in its
// OWN isolated remote directory (`project-<projectId>`). Additional sources
// never get the anchor's git-history/overlay semantics. Per-project failure
// isolation: one project's failure logs a warning and is skipped; the run and
// the other projects continue (no workspace restore, unlike an asset failure).
const additionalSourceDirs: Record<string, string> = {};
for (const source of input.additionalSources ?? []) {
const { localPath, projectId, ignoreResolution } = source;
try {
if (!path.posix.isAbsolute(localPath)) {
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
}
if (
projectId.length === 0 ||
projectId.includes("/") ||
projectId.includes("\\") ||
projectId.includes("..")
) {
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
}
// Fail closed: a project whose ignore resolution failed is not staged at
// all — the existing per-project skip-and-warn path below handles it.
if (ignoreResolution.kind === "failed") {
throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`);
}
const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`);
const exclude = mergeExcludes(
REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
referencedSourceIgnoreExcludeEntries(ignoreResolution),
);
await syncDirectoryToSsh({
spec: input.spec,
localDir: localPath,
remoteDir,
exclude,
onProgress: input.onProgress,
progressLabel: `project-${projectId}`,
});
additionalSourceDirs[projectId] = remoteDir;
} catch (error) {
console.warn(
`[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,
);
}
}
const restore = async (onProgress?: RuntimeProgressSink) => {
let failure: unknown;
try {
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
spec: input.spec,
localDir: input.workspaceLocalDir,
remoteDir: workspaceRemoteDir,
baselineSnapshot,
restoreGitHistory: preparedWorkspace.gitBacked,
onProgress,
repositories: repositoryBaselines,
});
} catch (error) { failure = error; }
for (const asset of input.assets ?? []) {
if (!asset.restore) continue;
try { await asset.restore({
assetDir: path.posix.join(runtimeRootDir, asset.key),
readFile: (remotePath) => readRemoteFile(input.spec, remotePath),
}); } catch (error) { failure ??= error; }
}
if (!input.retainRuntime) {
try { await cleanupPrivateRuntime(); } catch (error) { failure ??= error; }
}
if (failure) throw failure;
};
let privateRestore: Promise<void> | undefined;
return {
spec: input.spec,
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
runtimeRootDir,
assetDirs,
additionalSourceDirs,
disposeRuntime: cleanupPrivateRuntime,
restoreWorkspace: (onProgress?: RuntimeProgressSink) => {
if (!input.privateRuntime || input.retainRuntime) return restore(onProgress);
return privateRestore ??= restore(onProgress);
},
};
}