mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-18 14:35:55 +01:00
Stage the selected agent account in a private remote home and restore refreshed credentials before cleanup. Preserve remote authentication for existing SSH environments and expose it as an advanced option. Co-Authored-By: Paperclip <noreply@paperclip.ing>
305 lines
11 KiB
TypeScript
305 lines
11 KiB
TypeScript
import path from "node:path";
|
|
import { GIT_ARCHIVE_EXCLUDES, PROJECT_REPOSITORIES_DIR } from "./git-workspace-sync.js";
|
|
import {
|
|
type SshRemoteExecutionSpec,
|
|
prepareWorkspaceForSshExecution,
|
|
runSshCommand,
|
|
restoreWorkspaceFromSshExecution,
|
|
syncDirectoryToSsh,
|
|
} from "./ssh.js";
|
|
import {
|
|
mergeExcludes,
|
|
referencedSourceIgnoreExcludeEntries,
|
|
type SandboxAdditionalSource,
|
|
type SandboxManagedRuntimeAssetRestoreContext,
|
|
} from "./sandbox-managed-runtime.js";
|
|
import { captureDirectorySnapshot, type DirectorySnapshot } from "./workspace-restore-merge.js";
|
|
import type { RuntimeProgressSink } from "./runtime-progress.js";
|
|
|
|
// The fixed heavy-directory excludes every referenced project drops,
|
|
// regardless of its ignore resolution. A `git`-resolved project additionally
|
|
// drops its own resolved ignored paths (see `referencedSourceIgnoreExcludeEntries`
|
|
// and the per-project merge below); an `other` project keeps only this set.
|
|
const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [
|
|
"node_modules",
|
|
"vendor",
|
|
"dist",
|
|
"build",
|
|
"out",
|
|
"coverage",
|
|
".next",
|
|
".turbo",
|
|
".cache",
|
|
".git",
|
|
].flatMap((entry) => [entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`]);
|
|
|
|
export interface RemoteManagedRuntimeAsset {
|
|
key: string;
|
|
localDir: string;
|
|
followSymlinks?: boolean;
|
|
exclude?: string[];
|
|
restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise<void>;
|
|
}
|
|
|
|
export interface PreparedRemoteManagedRuntime {
|
|
spec: SshRemoteExecutionSpec;
|
|
workspaceLocalDir: string;
|
|
workspaceRemoteDir: string;
|
|
runtimeRootDir: string;
|
|
assetDirs: Record<string, string>;
|
|
/**
|
|
* Remote directory of each additional (referenced) project that staged
|
|
* successfully, keyed by `projectId`. A project whose staging failed is
|
|
* absent (per-project failure isolation).
|
|
*/
|
|
additionalSourceDirs: Record<string, string>;
|
|
disposeRuntime(): Promise<void>;
|
|
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
|
|
}
|
|
|
|
function asObject(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function asString(value: unknown): string {
|
|
return typeof value === "string" ? value : "";
|
|
}
|
|
|
|
function asNumber(value: unknown): number {
|
|
return typeof value === "number" ? value : Number(value);
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
|
}
|
|
|
|
async function readRemoteFile(spec: SshRemoteExecutionSpec, remotePath: string): Promise<Buffer> {
|
|
const result = await runSshCommand(spec, `base64 < ${shellQuote(remotePath)}`, {
|
|
maxBuffer: 1024 * 1024,
|
|
});
|
|
return Buffer.from(result.stdout.replace(/\s+/g, ""), "base64");
|
|
}
|
|
|
|
export function buildRemoteExecutionSessionIdentity(spec: SshRemoteExecutionSpec | null) {
|
|
if (!spec) return null;
|
|
return {
|
|
transport: "ssh",
|
|
...(typeof spec.useHostAuth === "boolean" ? { useHostAuth: spec.useHostAuth } : {}),
|
|
host: spec.host,
|
|
port: spec.port,
|
|
username: spec.username,
|
|
remoteCwd: spec.remoteCwd,
|
|
} as const;
|
|
}
|
|
|
|
export function remoteExecutionSessionMatches(saved: unknown, current: SshRemoteExecutionSpec | null): boolean {
|
|
const currentIdentity = buildRemoteExecutionSessionIdentity(current);
|
|
if (!currentIdentity) return false;
|
|
|
|
const parsedSaved = asObject(saved);
|
|
return (
|
|
asString(parsedSaved.transport) === currentIdentity.transport &&
|
|
parsedSaved.useHostAuth === currentIdentity.useHostAuth &&
|
|
asString(parsedSaved.host) === currentIdentity.host &&
|
|
asNumber(parsedSaved.port) === currentIdentity.port &&
|
|
asString(parsedSaved.username) === currentIdentity.username &&
|
|
asString(parsedSaved.remoteCwd) === currentIdentity.remoteCwd
|
|
);
|
|
}
|
|
|
|
export async function prepareRemoteManagedRuntime(input: {
|
|
spec: SshRemoteExecutionSpec;
|
|
runId: string;
|
|
adapterKey: string;
|
|
workspaceLocalDir: string;
|
|
workspaceRemoteDir?: string;
|
|
syncWorkspace?: boolean;
|
|
privateRuntime?: boolean;
|
|
retainRuntime?: boolean;
|
|
workspaceFileMode?: "all";
|
|
workspaceExclude?: string[];
|
|
assets?: RemoteManagedRuntimeAsset[];
|
|
/** Referenced (additional) projects to stage as plain, read-only trees. */
|
|
additionalSources?: SandboxAdditionalSource[];
|
|
// Upload progress sink. Threaded for the byte-counting transport rewrite; the
|
|
// child task wires it into the workspace/asset transfers.
|
|
onProgress?: RuntimeProgressSink;
|
|
}): Promise<PreparedRemoteManagedRuntime> {
|
|
const baseWorkspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
|
|
const syncWorkspace = input.syncWorkspace !== false;
|
|
const workspaceRemoteDir = syncWorkspace
|
|
? path.posix.join(
|
|
baseWorkspaceRemoteDir,
|
|
".paperclip-runtime",
|
|
"runs",
|
|
input.runId,
|
|
"workspace",
|
|
)
|
|
: baseWorkspaceRemoteDir;
|
|
let runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
|
|
let privateRuntimeDir: string | null = null;
|
|
const cleanupPrivateRuntime = async () => {
|
|
if (!privateRuntimeDir) return;
|
|
await runSshCommand(input.spec, `rm -rf -- ${shellQuote(privateRuntimeDir)}`, { timeoutMs: 15_000 });
|
|
privateRuntimeDir = null;
|
|
};
|
|
|
|
const preparedWorkspace = syncWorkspace
|
|
? await prepareWorkspaceForSshExecution({
|
|
spec: input.spec,
|
|
localDir: input.workspaceLocalDir,
|
|
remoteDir: workspaceRemoteDir,
|
|
onProgress: input.onProgress,
|
|
workspaceFileMode: input.workspaceFileMode,
|
|
workspaceExclude: input.workspaceExclude,
|
|
})
|
|
: null;
|
|
const projectRepositories = preparedWorkspace?.repositories ?? [];
|
|
const baselineSnapshot = preparedWorkspace
|
|
? await captureDirectorySnapshot(input.workspaceLocalDir, {
|
|
exclude: preparedWorkspace.gitBacked
|
|
? [
|
|
...GIT_ARCHIVE_EXCLUDES,
|
|
".paperclip-runtime",
|
|
...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : preparedWorkspace.excludedPaths ?? []),
|
|
]
|
|
: [".paperclip-runtime", ...(preparedWorkspace.excludedPaths ?? []), ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])],
|
|
})
|
|
: null;
|
|
const repositoryBaselines: Array<{ path: string; baselineSnapshot: DirectorySnapshot }> = [];
|
|
for (const repository of projectRepositories) {
|
|
repositoryBaselines.push({
|
|
path: repository,
|
|
baselineSnapshot: await captureDirectorySnapshot(path.join(input.workspaceLocalDir, repository), {
|
|
exclude: [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"],
|
|
}),
|
|
});
|
|
}
|
|
|
|
const assetDirs: Record<string, string> = {};
|
|
try {
|
|
if (input.privateRuntime) {
|
|
const result = await runSshCommand(input.spec, "umask 077; mktemp -d /tmp/paperclip-ssh-runtime.XXXXXXXXXX", { timeoutMs: 15_000 });
|
|
const directory = result.stdout.trim();
|
|
if (!/^\/tmp\/paperclip-ssh-runtime\.[A-Za-z0-9]+$/.test(directory)) throw new Error("SSH private runtime directory unavailable");
|
|
runtimeRootDir = privateRuntimeDir = directory;
|
|
}
|
|
for (const asset of input.assets ?? []) {
|
|
if (input.privateRuntime && !/^[A-Za-z0-9_-]+$/.test(asset.key)) throw new Error("Invalid SSH runtime asset key");
|
|
const remoteDir = path.posix.join(runtimeRootDir, asset.key);
|
|
assetDirs[asset.key] = remoteDir;
|
|
await syncDirectoryToSsh({
|
|
spec: input.spec,
|
|
localDir: asset.localDir,
|
|
remoteDir,
|
|
followSymlinks: asset.followSymlinks,
|
|
exclude: asset.exclude,
|
|
onProgress: input.onProgress,
|
|
progressLabel: asset.key,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
try {
|
|
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
|
|
spec: input.spec,
|
|
localDir: input.workspaceLocalDir,
|
|
remoteDir: workspaceRemoteDir,
|
|
baselineSnapshot,
|
|
restoreGitHistory: preparedWorkspace.gitBacked,
|
|
onProgress: input.onProgress,
|
|
repositories: repositoryBaselines,
|
|
});
|
|
} finally { await cleanupPrivateRuntime(); }
|
|
throw error;
|
|
}
|
|
|
|
// Stage each referenced (additional) project as a plain, read-only tree in its
|
|
// OWN isolated remote directory (`project-<projectId>`). Additional sources
|
|
// never get the anchor's git-history/overlay semantics. Per-project failure
|
|
// isolation: one project's failure logs a warning and is skipped; the run and
|
|
// the other projects continue (no workspace restore, unlike an asset failure).
|
|
const additionalSourceDirs: Record<string, string> = {};
|
|
for (const source of input.additionalSources ?? []) {
|
|
const { localPath, projectId, ignoreResolution } = source;
|
|
try {
|
|
if (!path.posix.isAbsolute(localPath)) {
|
|
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
|
|
}
|
|
if (
|
|
projectId.length === 0 ||
|
|
projectId.includes("/") ||
|
|
projectId.includes("\\") ||
|
|
projectId.includes("..")
|
|
) {
|
|
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
|
|
}
|
|
// Fail closed: a project whose ignore resolution failed is not staged at
|
|
// all — the existing per-project skip-and-warn path below handles it.
|
|
if (ignoreResolution.kind === "failed") {
|
|
throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`);
|
|
}
|
|
const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`);
|
|
const exclude = mergeExcludes(
|
|
REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
|
|
referencedSourceIgnoreExcludeEntries(ignoreResolution),
|
|
);
|
|
await syncDirectoryToSsh({
|
|
spec: input.spec,
|
|
localDir: localPath,
|
|
remoteDir,
|
|
exclude,
|
|
onProgress: input.onProgress,
|
|
progressLabel: `project-${projectId}`,
|
|
});
|
|
additionalSourceDirs[projectId] = remoteDir;
|
|
} catch (error) {
|
|
console.warn(
|
|
`[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const restore = async (onProgress?: RuntimeProgressSink) => {
|
|
let failure: unknown;
|
|
try {
|
|
if (preparedWorkspace && baselineSnapshot) await restoreWorkspaceFromSshExecution({
|
|
spec: input.spec,
|
|
localDir: input.workspaceLocalDir,
|
|
remoteDir: workspaceRemoteDir,
|
|
baselineSnapshot,
|
|
restoreGitHistory: preparedWorkspace.gitBacked,
|
|
onProgress,
|
|
repositories: repositoryBaselines,
|
|
});
|
|
} catch (error) { failure = error; }
|
|
for (const asset of input.assets ?? []) {
|
|
if (!asset.restore) continue;
|
|
try { await asset.restore({
|
|
assetDir: path.posix.join(runtimeRootDir, asset.key),
|
|
readFile: (remotePath) => readRemoteFile(input.spec, remotePath),
|
|
}); } catch (error) { failure ??= error; }
|
|
}
|
|
if (!input.retainRuntime) {
|
|
try { await cleanupPrivateRuntime(); } catch (error) { failure ??= error; }
|
|
}
|
|
if (failure) throw failure;
|
|
};
|
|
let privateRestore: Promise<void> | undefined;
|
|
|
|
return {
|
|
spec: input.spec,
|
|
workspaceLocalDir: input.workspaceLocalDir,
|
|
workspaceRemoteDir,
|
|
runtimeRootDir,
|
|
assetDirs,
|
|
additionalSourceDirs,
|
|
disposeRuntime: cleanupPrivateRuntime,
|
|
restoreWorkspace: (onProgress?: RuntimeProgressSink) => {
|
|
if (!input.privateRuntime || input.retainRuntime) return restore(onProgress);
|
|
return privateRestore ??= restore(onProgress);
|
|
},
|
|
};
|
|
}
|