import type { WorkspaceCheckpointMetrics } from "./workspace-checkpoint.js"; import { workspaceSeedGeneration, workspaceSeedCloneSnapshot, readWorkspaceSeedGeneration, publishWorkspaceSeedGeneration } from "./workspace-seed-cache.js"; import { writeCheckpointBaseline, workspaceCheckpointScript, readCheckpointSnapshot, validateCheckpointPayload } from "./workspace-checkpoint.js"; import { execFile as execFileCallback } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { constants as fsConstants, createReadStream, promises as fs, } from "node:fs"; import os from "node:os"; import { createWorkspaceManifest, WorkspaceManifestMap, workspacePaths, workspacePathMatcher, writeWorkspacePaths, isPathManifest, type WorkspacePaths } from "./workspace-manifest.js"; import path from "node:path"; import { promisify } from "node:util"; import { buildRemoteGitDeltaBundleScript, GIT_SHALLOW_METADATA_EXIT_CODE, GIT_SHALLOW_MAX_BYTES, GitShallowMetadataError, isMissingGitPrerequisiteError, createImportedGitRef, createRemoteGitExportRef, deleteLocalGitRef, fetchGitBundleIntoLocalRef, GIT_ARCHIVE_EXCLUDES, buildRefreshWorkspaceGitIndexCommand, type GitWorkspaceSnapshot, integrateImportedGitHead, readGitWorkspaceSnapshot, readManagedWorkspaceRepositories, copyCloneTree, disposeGitWorkspaceSnapshot, ReferencedSourceIgnoreScanLimitExceededError, readReferencedSourceGitIgnoredPaths, resetLocalGitIndexToHead, withShallowGitWorkspaceClone, WORKSPACE_GIT_SCAN_SATURATED_CODE, } from "./git-workspace-sync.js"; import { captureDirectorySnapshot, captureDirectorySnapshotOffThread, directorySnapshotSha256, classifyWorkspaceRestoreFailure, disposeDirectorySnapshot, selectDirectorySnapshot, mergeDirectoryWithBaseline, type DirectorySnapshot, } from "./workspace-restore-merge.js"; import { createRuntimeProgressReporter, type RuntimeProgressDirection, type RuntimeProgressPhase, type RuntimeProgressSink, type RuntimeStatusPhase, type RuntimeStatusSink, } from "./runtime-progress.js"; import { isRelativePathOrDescendant, shouldExcludePath } from "./exclude-patterns.js"; import { scheduleSyncOperations, SYNC_OPERATION_CONCURRENCY_LIMIT, type SyncOperationTask, } from "./sync-operation-schedule.js"; import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js"; import { recordWorkspaceRestoreDiagnostic, withWorkspaceRestoreDiagnostics, withWorkspaceRestoreStep, type WorkspaceRestoreDiagnostic, } from "./workspace-restore-diagnostics.js"; const execFile = promisify(execFileCallback); const SANDBOX_WORKSPACE_HEAVY_DIR_NAMES = [ "node_modules", "vendor", "dist", "build", "out", "coverage", ".next", ".turbo", ".cache", ] as const; const SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES = SANDBOX_WORKSPACE_HEAVY_DIR_NAMES.flatMap((entry) => [ entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`, ]); export interface SandboxRemoteExecutionSpec { transport: "sandbox"; provider: string; sandboxId: string; remoteCwd: string; timeoutMs: number; apiKey: string | null; } /** * Remote paths handed to an asset's `provision.postUploadCommand`. All are POSIX * paths inside the sandbox: `assetTarPath` is the uploaded asset tarball, * `assetDir` is where the asset should be materialized, and `runtimeRootDir` * is the directory any `stageFiles` were written into. */ export interface SandboxManagedRuntimeAssetProvisionContext { assetTarPath: string; assetDir: string; runtimeRootDir: string; } /** * Per-asset inbound provisioning contribution. The core is adapter-agnostic: * an asset that supplies neither `stageFiles` nor `postUploadCommand` is * materialized with a plain destroy-then-replace `tar -xf`. An adapter that * needs custom provisioning (e.g. a credential merge) supplies helper files via * `stageFiles` and the shell command that consumes them via `postUploadCommand`. * * Both contributions ride the unified {@link SandboxSyncOperation} the core * builds per asset: `stageFiles` become additional `files` mappings placed * alongside the asset tar, and `postUploadCommand` becomes the operation's * ordered `postUploadCommands`. See {@link SandboxPostUploadCommand} for the * command-origin / confinement security contract (C1–C3). */ export interface SandboxManagedRuntimeAssetProvision { /** * Extra files placed into `runtimeRootDir` (alongside the asset tar) before * the post-upload command runs — typically helper scripts the command * invokes. Contents may be raw bytes or a UTF-8 string. */ stageFiles?: { name: string; contents: Buffer | string }[]; /** * Builds the opaque, adapter-authored shell command that materializes the * uploaded asset tar into `assetDir`, run as the operation's ordered * post-upload command after every mapping has landed. Defaults to a plain * destroy-then-replace `tar -xf` extraction when omitted. Any path embedded in * the command MUST be built from already-confined paths and shell-quoted (C3). */ postUploadCommand?: (ctx: SandboxManagedRuntimeAssetProvisionContext) => string; } /** * Context passed to an asset's `restore` contribution during teardown. * `assetDir` is the asset's directory inside the sandbox and `readFile` reads * a file back from the sandbox as raw bytes. `tempDir` is a host scratch * directory that belongs to this restore task alone. The shared scheduler can * run restore tasks at the same time, so a task must not share scratch space * with another task. A restore that needs a host temporary file writes it under * `tempDir`. The coordinator removes the directory after the task settles. The * sandbox coordinator always sets `tempDir`. A serial runtime that never runs * restore tasks at the same time can omit it. */ export interface SandboxManagedRuntimeAssetRestoreContext { assetDir: string; readFile: (remotePath: string) => Promise; tempDir?: string; } export interface SandboxManagedRuntimeAsset { key: string; localDir: string; followSymlinks?: boolean; exclude?: string[]; /** Optional inbound provisioning contribution (staged files + extract command). */ provision?: SandboxManagedRuntimeAssetProvision; /** * Optional teardown/outbound contribution, invoked once per asset during * `restoreWorkspace`. Defaults to a no-op when omitted. */ restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise; } /** * How a referenced project's Git-ignored paths were resolved, computed once * per project by `resolveReferencedSourceIgnore` before staging starts. The * sandbox lane, the SSH lane, and the content-signature walk each consume * this ONE resolution, so the three sites never drift apart. * * - `git`: `localPath` is a Git work tree. `ignoredPaths` are its ignored * entries, already re-relativized to `localPath` (see * `resolveReferencedSourceIgnore`). * - `other`: `localPath` is not a Git work tree. The staging path keeps * today's fixed heavy-directory excludes. * - `failed`: the Git read failed, timed out, breached a parse bound, or * returned output the resolver could not safely re-relativize. The project * is NOT staged (fail closed) — every site records it as a per-project * failure instead of shipping it unfiltered. `reason` is always one of * {@link REFERENCED_SOURCE_IGNORE_FAILURE_REASONS} — never a raw Git or tar * diagnostic, an absolute host path, or a basename. */ export type ReferencedSourceIgnoreResolution = | { kind: "git"; ignoredPaths: string[] } | { kind: "other" } | { kind: "failed"; reason: string }; /** * The fixed, allowlisted failure categories a `failed` * {@link ReferencedSourceIgnoreResolution} reports as `reason`. This is the * ENTIRE vocabulary: no absolute host path, no basename, no opaque token, and * no raw Git or tar stderr ever reaches `reason` — only one of these three * stable strings, chosen once at the single construction point in * `resolveReferencedSourceIgnore`. A `failed` resolution always prevents * staging and is always re-resolved before its next use, so two different * underlying failures colliding on the same category (e.g. a timeout and a * malformed-output error both reporting `scanFailed`) never weakens the * fail-closed decision. */ export const REFERENCED_SOURCE_IGNORE_FAILURE_REASONS = { /** A Git read failed, timed out, was cancelled, or returned malformed output — including a saturated scan queue that never recovered after its retries. */ scanFailed: "git-ignore-scan-failed", /** The parsed ignored-entry count or total UTF-8 byte size breached its bound (see `readReferencedSourceGitIgnoredPaths`). */ limitExceeded: "git-ignore-scan-limit-exceeded", /** The referenced project's `localPath` is not a descendant of its own Git top level. */ toplevelNotDescendant: "git-toplevel-not-descendant", } as const; /** * A referenced (additional) project to stage into the run sandbox as a plain, * read-only tree. `localPath` is the host checkout directory. Upstream code * already authorized and realized this directory (`project:read`); this layer * adds no authorization logic. `projectId` names the isolated remote * subdirectory (`project-` under the runtime root) the tree lands in. * * Additional sources are plain trees only. They never carry the anchor * workspace's git-history, overlay, or `.paperclip-runtime` preservation * semantics — those stay anchor-only. * * `ignoreResolution` is required so every construction site must supply it * explicitly — a caller cannot default to the unfiltered legacy behavior by * omission. Resolve it once per project with `resolveReferencedSourceIgnore`. */ export interface SandboxAdditionalSource { localPath: string; projectId: string; ignoreResolution: ReferencedSourceIgnoreResolution; } /** * Escape tar `--exclude` glob metacharacters (`*`, `?`, `[`) in a literal * path, so a Git-ignored path that happens to contain one of them is matched * literally instead of as a pattern. Without this, a repository-controlled * path containing e.g. `*` could exclude unrelated sibling files that * happen to match the resulting glob. GNU tar and bsdtar both honor a * backslash as a `fnmatch` escape character, so this is not command * injection — `createTarballFromDirectory` and the SSH tar equivalent both * pass `--exclude` values as argument-vector entries, never through a shell. */ export function escapeTarExcludeLiteral(entry: string): string { return entry.replace(/\\/g, "\\\\").replace(/([*?[])/g, "\\$1"); } /** * The tar `--exclude` entries a referenced project's resolved ignore set * contributes, on top of the fixed heavy-directory excludes every site * already applies. Empty for `other` (today's fixed excludes are enough) * and for `failed` (the project is not staged at all, so no exclude list * matters). */ export function referencedSourceIgnoreExcludeEntries(resolution: ReferencedSourceIgnoreResolution): string[] { return resolution.kind === "git" ? resolution.ignoredPaths.map(escapeTarExcludeLiteral) : []; } /** * Compute the relative position of `localPath` under a Git `toplevel` * directory, as a POSIX path with no leading or trailing slash. Returns `""` * when `localPath` IS the toplevel. Returns `null` when the relation is not a * plain descendant — `localPath` escapes upward from `toplevel`, resolves to * an absolute/rooted result (a different filesystem root), or the two paths * are otherwise not comparable. The caller treats `null` as a resolution * failure (fail closed), never as "nothing to exclude". */ /** * Physical (symlink-resolved) form of a path, falling back to the plain * resolved form when realpath fails (e.g. the path vanished mid-run — the * downstream descendant check then fails closed on the string form). * Git prints physical toplevels, so both sides of the descendant comparison * must be physical too; comparing a logical path against a physical one makes * any symlinked project or temp directory (macOS `/var` → `/private/var`) * fail resolution spuriously. */ async function physicalPath(input: string): Promise { try { return await fs.realpath(input); } catch { return path.resolve(input); } } function relativizeUnderGitToplevel(input: { toplevel: string; localPath: string }): string | null { const toplevel = path.resolve(input.toplevel); const localPath = path.resolve(input.localPath); const relative = path.relative(toplevel, localPath); if (relative === "") return ""; if (path.isAbsolute(relative)) return null; if (relative === ".." || relative.startsWith(`..${path.sep}`)) return null; return relative.split(path.sep).join("/"); } /** * Re-relativize root-relative ignored paths (as `readReferencedSourceGitIgnoredPaths` * reports them, from the repository toplevel) to `offset`, the position of * the referenced project's `localPath` under that toplevel. Keeps only the * entries that are `offset` itself or a descendant of it — an ignored path * elsewhere in the repository does not apply to this project's staged tree — * and strips the `offset` prefix so the result matches the tar member * namespace, which is `localPath`-relative. */ function reRelativizeIgnoredPathsToLocalPath(input: { ignoredPaths: string[]; offset: string }): string[] { if (input.offset === "") { return [...input.ignoredPaths]; } const prefix = `${input.offset}/`; return input.ignoredPaths .filter((entry) => entry.startsWith(prefix)) .map((entry) => entry.slice(prefix.length)) .filter(Boolean); } /** * Bounded backoff before each retry of a saturated Git scan: none before the * first attempt, 1 second before the second, 2 seconds before the third. Three * total attempts (the first plus these two retries) is a liveness parameter, * not a security control — the retry only ever fires for the scheduler's * typed saturation code (see {@link isWorkspaceGitScanSaturatedError}). */ const REFERENCED_SOURCE_IGNORE_SCAN_RETRY_DELAYS_MS = [1_000, 2_000] as const; async function delay(ms: number): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } /** * Backoff before each automatic retry of a workspace copy-back: three total * attempts. A failed copy-back otherwise retains the sandbox for board repair, * so a dropped transfer or a busy merge lock must not end the run's restore. */ const WORKSPACE_RESTORE_RETRY_DELAYS_MS = [2_000, 5_000] as const; /** * Retry a copy-back attempt only while it has written nothing into the host * workspace. The baseline stays valid until then, so a fresh attempt is the * same merge. Never retry an unsafe archive or a permission failure, and never * retry once the merge has started writing host files. */ async function retryWorkspaceRestoreBeforeHostWrite( delaysMs: readonly number[], attempt: (markHostWriteStarted: () => void) => Promise, ): Promise { for (let retry = 0; ; retry++) { let hostWriteStarted = false; try { await attempt(() => { hostWriteStarted = true; }); return; } catch (error) { const code = classifyWorkspaceRestoreFailure(error); const retryable = code === "restore_failed" || code === "restore_lock_timeout"; if (hostWriteStarted || !retryable || retry >= delaysMs.length) throw error; await delay(delaysMs[retry]!); } } } /** * True only when `error` carries the workspace Git scan scheduler's typed * saturation code on its `code` property. Matches the code alone, never * message text — a message can change wording without changing meaning, and * matching text would silently stop retrying (or start retrying the wrong * failure) the moment it did. */ function isWorkspaceGitScanSaturatedError(error: unknown): boolean { return ( typeof error === "object" && error !== null && "code" in error && (error as { code?: unknown }).code === WORKSPACE_GIT_SCAN_SATURATED_CODE ); } function isCheckpointCaptureCapacityError(error: unknown): boolean { if (!error || typeof error !== "object") return false; const failure = error as { code?: unknown; message?: unknown; stderr?: unknown }; if (["ENOSPC", "EDQUOT", "SQLITE_FULL"].includes(String(failure.code))) return true; const diagnostic = [failure.message, failure.stderr].filter(value => typeof value === "string").join("\n"); return /Workspace checkpoint storage is below its free-space reserve|\b(?:ENOSPC|EDQUOT|SQLITE_FULL)\b|database or disk is full/i.test(diagnostic); } /** * Resolve a referenced project's Git-ignored paths ONCE, before any staging * site runs. Called once per project (see `execute.ts`); the sandbox lane, * the SSH lane, and the content-signature walk all consume this one result, * so they can never apply a different exclusion set to the same project. * * Fails closed: a Git read error, a timeout, malformed output, a parse-bound * breach, or a `localPath` that is not a plain descendant of its own Git * toplevel all return `failed`, never an empty ignore list — an empty list * means "resolved, nothing extra to exclude", which is a different claim than * "the resolution did not run". * * Retries ONLY a saturated scan queue (the shared workspace Git operation * scheduler rejecting before spawn because it is at capacity) — a liveness * condition, not an integrity one. Three attempts total, with the bounded * backoff in {@link REFERENCED_SOURCE_IGNORE_SCAN_RETRY_DELAYS_MS}, retried * through the same registered scheduler every time. No direct-spawn fallback * exists: bypassing the scheduler would defeat the process-wide concurrency * limit it enforces. Every other failure — timeout, cancellation, an output * limit, a permission error, malformed output, a real Git failure, or a bound * breach — makes exactly one attempt and fails closed immediately. */ export async function resolveReferencedSourceIgnore(localPath: string): Promise { let scan: Awaited> = null; let failureReason: string | null = null; for (let attempt = 0; attempt <= REFERENCED_SOURCE_IGNORE_SCAN_RETRY_DELAYS_MS.length; attempt += 1) { try { scan = await readReferencedSourceGitIgnoredPaths(localPath); failureReason = null; break; } catch (error) { failureReason = error instanceof ReferencedSourceIgnoreScanLimitExceededError ? REFERENCED_SOURCE_IGNORE_FAILURE_REASONS.limitExceeded : REFERENCED_SOURCE_IGNORE_FAILURE_REASONS.scanFailed; const isLastAttempt = attempt === REFERENCED_SOURCE_IGNORE_SCAN_RETRY_DELAYS_MS.length; if (isLastAttempt || !isWorkspaceGitScanSaturatedError(error)) { break; } await delay(REFERENCED_SOURCE_IGNORE_SCAN_RETRY_DELAYS_MS[attempt]!); } } if (failureReason !== null) { return { kind: "failed", reason: failureReason }; } if (!scan) { return { kind: "other" }; } const offset = relativizeUnderGitToplevel({ toplevel: await physicalPath(scan.toplevel), localPath: await physicalPath(localPath), }); if (offset === null) { return { kind: "failed", reason: REFERENCED_SOURCE_IGNORE_FAILURE_REASONS.toplevelNotDescendant }; } return { kind: "git", ignoredPaths: reRelativizeIgnoredPathsToLocalPath({ ignoredPaths: scan.ignoredPaths, offset }), }; } /** * Per-call byte-level progress hook. `transferredBytes`/`totalBytes` are decoded * file bytes (not the base64 wire size). `totalBytes` is null when the size is * not known up front. The transport is the source of truth for byte counts; the * orchestrator owns the phase label and direction. */ export interface SandboxTransferProgressOptions { onProgress?: (transferredBytes: number, totalBytes: number | null) => void | Promise; } /** * A single source→target file or directory transfer within a sync operation. * Mirrors the plugin SDK `PluginSyncFileMapping`; kept as a local structural * type so `adapter-utils` does not depend on the plugin SDK. For `syncIn`, * `sourcePath` is a host path and `targetPath` a sandbox path; for `syncOut` the * direction is reversed. Sandbox paths are POSIX. */ export interface SandboxSyncFileMapping { sourcePath: string; targetPath: string; kind: "file" | "directory"; mode?: number; exclude?: string[]; followSymlinks?: boolean; /** * Advisory read-write intent for the sandbox target. `"rw"` marks a target the * agent may change and keep; `"ro"` marks a read-only tree. An absent value * defaults to `"ro"` (read-only is the safe default for an advisory signal). * The field is advisory metadata for an optional sandbox feedback wrapper. It * does not change the transfer and adds no security. */ access?: "rw" | "ro"; /** * The sandbox directory that becomes read-write when `access` is `"rw"` and a * post-upload command extracts `targetPath` into a different directory. A tar * mapping uploads an archive under the runtime root, so its `targetPath` is the * staging archive, not the directory the extract command fills. This field * names that final destination directory. When absent, the read-write * destination is the parent directory of `targetPath`. Advisory; ignored when * `access` is not `"rw"`. */ writablePath?: string; } /** * A control command run against the sandbox after a sync operation's files have * landed. Mirrors the plugin SDK `PluginPostUploadCommand`; kept as a local * structural type so `adapter-utils` does not depend on the plugin SDK. Ordered * within {@link SandboxSyncOperation.postUploadCommands} and executed in array * order, fail-fast (first non-zero exit or timeout aborts the operation). * * SECURITY — command origin (Stage-1 design review, condition C1). `command` is * a **Paperclip/adapter-authored control operation**: it may be supplied ONLY by * core/adapter code. No server route, issue/comment content, project/workspace * file content, provider-plugin callback, or arbitrary adapter config may supply * a raw `command` string; any path embedded in it MUST be built by adapter/core * helpers from already-confined paths and shell-quoted (C3). Providers treat the * command as **opaque** — execute or reject, never rewrite/concatenate/append. */ export interface SandboxPostUploadCommand { /** The opaque, adapter-authored shell command to run after upload. */ command: string; /** * Working directory for the command. When present, MUST be an absolute POSIX * path confined under the operation's allowed sandbox target root (C2). When * absent, defaults to the runtime's stable command cwd — never a process * default cwd. */ cwd?: string; /** Optional per-command timeout in milliseconds. */ timeoutMs?: number; } /** * An ordered, opaque unit of work handed to the native sync transport. The * `operationId` is an opaque, non-sensitive token authored by the orchestrator * (never a caller/asset identifier that could leak intent); a provider MUST NOT * interpret it. */ export interface SandboxSyncOperation { operationId: string; files: SandboxSyncFileMapping[]; /** * Optional ordered control commands run after this operation's files land, in * array order, fail-fast. Absent means "no commands" — byte-identical to a * pre-contract operation. See {@link SandboxPostUploadCommand} for the command * origin/confinement security contract (C1–C4). */ postUploadCommands?: SandboxPostUploadCommand[]; } export interface SandboxSyncResult { operations: { operationId: string; filesTransferred: number; bytesTransferred: number }[]; } export interface SandboxManagedRuntimeClient { makeDir(remotePath: string): Promise; writeFile(remotePath: string, bytes: ArrayBuffer, options?: SandboxTransferProgressOptions): Promise; readFile( remotePath: string, options?: SandboxTransferProgressOptions, ): Promise; listFiles(remotePath: string): Promise; remove(remotePath: string): Promise; run(command: string, options: { timeoutMs: number }): Promise; /** * True when the orchestrator may run this client's sync operations * concurrently. The base64 fallback always sets it true. A native provider * takes the value from the verified `concurrentSyncOperations` opt-in; an * undeclared native provider keeps it false. One flag serves both `syncIn` and * `syncOut`. `createCommandManagedRuntimeClient` always sets it on a prepared * client; it is optional here so a test mock can omit it. */ allowConcurrentSyncOperations?: boolean; /** * Optional native inbound transfer. Present only when the sandbox provider * advertises both `environmentSyncIn` and `environmentSyncOut`; otherwise the * orchestrator falls back to the tar + base64 `writeFile`/`run` path so * behavior is byte-identical to a provider that never opted in. */ syncIn?(operations: SandboxSyncOperation[]): Promise; /** Optional native outbound transfer. See {@link syncIn}. */ syncOut?(operations: SandboxSyncOperation[]): Promise; } /** * Host-side complete-mediation guard for native sync operations. The orchestrator * authors every `targetPath`, but the native transport crosses the host↔sandbox * trust boundary, so we canonicalize and confine each mapping's source and target * to an orchestrator-owned root before handing the operation to a provider. * Absolute escapes and `..` traversal are rejected fail-closed. Sandbox and host * paths on the server are POSIX. */ export function assertSyncOperationsConfined( operations: SandboxSyncOperation[], roots: { sourceRoots: string[]; targetRoots: string[] }, ): void { const confine = (candidate: string, allowed: string[], label: string): void => { const normalized = path.posix.normalize(candidate); if (!path.posix.isAbsolute(normalized) || normalized === ".." || normalized.includes("/../") || normalized.endsWith("/..")) { throw new Error(`sync operation ${label} path is not a confined absolute path: ${candidate}`); } const within = allowed.some((root) => { const normalizedRoot = path.posix.normalize(root); const prefix = normalizedRoot.endsWith("/") ? normalizedRoot : `${normalizedRoot}/`; return normalized === normalizedRoot || normalized.startsWith(prefix); }); if (!within) { throw new Error(`sync operation ${label} path escapes its confinement root: ${candidate}`); } }; for (const operation of operations) { for (const mapping of operation.files) { confine(mapping.sourcePath, roots.sourceRoots, "source"); confine(mapping.targetPath, roots.targetRoots, "target"); } } } export interface PreparedSandboxManagedRuntime { spec: SandboxRemoteExecutionSpec; workspaceLocalDir: string; workspaceRemoteDir: string; runtimeRootDir: string; assetDirs: Record; /** * Remote directory of each additional (referenced) project that staged * successfully, keyed by `projectId`. A project whose staging failed is * absent (per-project failure isolation). Empty when no additional sources * were requested. */ additionalSourceDirs: Record; /** * Each additional (referenced) project whose staging failed, paired with the * failure message. Per-project failure isolation keeps one project's failure * from aborting the run, so a failed project is absent from * `additionalSourceDirs` and present here. Empty when every requested project * staged, or when no additional sources were requested. */ additionalSourceFailures: AdditionalSourceStagingFailure[]; /** Durable merge inputs used to resume an outbound restore after host restart. */ workspaceSyncSnapshot: { baseline: DirectorySnapshot; gitSnapshot: GitWorkspaceSnapshot | null; repositories?: NonNullable; } | null; cleanupWorkspaceSnapshot(): Promise; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } export type WorkspaceInboundMode = "host_current" | "durable_seed" | "adopt_remote"; /** * Controller-owned archives for replaying the exact pre-turn workspace into a * replacement sandbox. Paths are never sent to the provider as credentials or * persisted in database metadata. */ export interface WorkspaceDurableSeedPaths { workspaceArchivePath: string; workspaceArchiveSha256?: string; gitArchivePath?: string | null; gitArchiveSha256?: string | null; } /** One additional (referenced) project that failed to stage into the sandbox. */ export interface AdditionalSourceStagingFailure { projectId: string; error: string; } function asObject(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } function asString(value: unknown): string { return typeof value === "string" ? value : ""; } function asNumber(value: unknown): number { return typeof value === "number" ? value : Number(value); } function shellQuote(value: string) { return `'${value.replace(/'/g, `'\"'\"'`)}'`; } function buildDefaultExtractRuntimeAssetCommand(input: { remoteAssetDir: string; remoteAssetTar: string; }): string { return `rm -rf ${shellQuote(input.remoteAssetDir)} && ` + `mkdir -p ${shellQuote(input.remoteAssetDir)} && ` + `tar -xf ${shellQuote(input.remoteAssetTar)} -C ${shellQuote(input.remoteAssetDir)} && ` + `rm -f ${shellQuote(input.remoteAssetTar)}`; } // Named builder (Security Condition C3): extract an uploaded workspace tarball // into `workspaceRemoteDir`, then remove the tarball. When `wipeExceptNames` is // present the target's direct children (except the preserved names) are removed // before extraction (destroy-then-replace); when null the tarball is overlaid on // top of the existing tree (e.g. a git overlay merge). Every path is // shell-quoted; no untrusted value is concatenated into the shell (C1/C3). function buildWorkspaceTarExtractCommand(input: { workspaceRemoteDir: string; remoteTar: string; wipeExceptNames: string[] | null; exclude?: string[]; }): string { // The wipe must also preserve any in-flight sync scratch tarball at the // workspace root. A concurrent referenced-project upload stages a scratch // tarball named `.paperclip-upload-.tar` there. Without this preserve // term the wipe unlinks the in-flight tarball and the later extract fails. // The static pattern must agree with the daytona scratch prefix // `SCRATCH_PREFIX` in // `packages/plugins/sandbox-providers/daytona/src/file-sync.ts:80`. // The term is a static literal; `preserveFindArgs` shell-quotes it, so the // shell passes it to `find -name` as a pattern (Security Conditions C1/C3). const wipe = input.wipeExceptNames ? ` && find ${shellQuote(input.workspaceRemoteDir)} -mindepth 1 -maxdepth 1 ` + `${preserveFindArgs([...input.wipeExceptNames, ".paperclip-upload-*"])} -exec rm -rf -- {} +` : ""; return ( `mkdir -p ${shellQuote(input.workspaceRemoteDir)}${wipe} && ` + `tar${(input.exclude ?? []).map(entry => ` --exclude=${shellQuote(escapeTarExcludeLiteral(entry))}`).join("")} -xf ${shellQuote(input.remoteTar)} -C ${shellQuote(input.workspaceRemoteDir)} && ` + `rm -f ${shellQuote(input.remoteTar)}` ); } // Named builder (C3): remove paths deleted in the host git worktree from the // sandbox workspace. Every path is shell-quoted; the caller supplies only // already-confined relative paths from the git snapshot. function buildRemoveDeletedPathsCommand(input: { remoteDir: string; manifestPath: string }): string { // NUL input plus bounded xargs batches preserve whitespace and never create const remove = `for entry do parent=$entry while [ "\${parent#*/}" != "$parent" ]; do parent=\${parent%/*} if [ -L "$parent" ] || { [ -e "$parent" ] && [ ! -d "$parent" ]; }; then exit 42; fi done done rm -rf -- "$@"`; return `cd ${shellQuote(input.remoteDir)} && xargs -0 -r -n 64 sh -c ${shellQuote(remove)} sh < ${shellQuote(input.manifestPath)} && rm -f -- ${shellQuote(input.manifestPath)}`; } async function bindWorkspaceSeedGitArchive(input: { sourcePath: string; targetPath: string; directory: string; git: GitWorkspaceSnapshot | null; repositories: NonNullable; }): Promise { await fs.copyFile(input.sourcePath, input.targetPath, fsConstants.COPYFILE_FICLONE); const entries: string[] = []; const bind = async (relative: string, snapshot: GitWorkspaceSnapshot) => { if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(snapshot.headCommit)) throw new Error("workspace_seed_git_head_invalid"); const headPath = path.posix.join(relative, ".git", "HEAD"); await fs.mkdir(path.dirname(path.join(input.directory, headPath)), { recursive: true }); if (snapshot.branchName === null) await fs.writeFile(path.join(input.directory, headPath), `${snapshot.headCommit}\n`); else { await execFile("git", ["check-ref-format", "--branch", snapshot.branchName]); const ref = `refs/heads/${snapshot.branchName}`; const refPath = path.posix.join(relative, ".git", ref); await fs.mkdir(path.dirname(path.join(input.directory, refPath)), { recursive: true }); await fs.writeFile(path.join(input.directory, refPath), `${snapshot.headCommit}\n`); await fs.writeFile(path.join(input.directory, headPath), `ref: ${ref}\n`); entries.push(refPath); } entries.push(headPath); for (const repository of snapshot.repositories ?? []) await bindChild(relative, repository); }; const bindChild = async (relative: string, repository: (typeof input.repositories)[number]) => { if (path.posix.isAbsolute(repository.path) || repository.path.split("/").includes("..")) throw new Error("workspace_seed_git_path_invalid"); await bind(path.posix.join(relative, repository.path), repository.snapshot); }; if (input.git) await bind("", input.git); else for (const repository of input.repositories) await bindChild("", repository); const list = path.join(input.directory, "entries.nul"); await fs.writeFile(list, entries.join("\0") + "\0"); await execTar(["-rf", input.targetPath, "-C", input.directory, "--null", "-T", list]); } function buildUniqueStagingPath(input: { targetPath: string; suffix: string }): string { return path.join(path.dirname(input.targetPath), `${input.suffix}.${randomUUID()}`); } // The workspace stages under `/workspace-upload.tar` and, for a // git-backed workspace, under `/git-workspace-upload.tar`. Each // asset stages under `/-upload.tar`, so an asset key equal to // one of these stems resolves to the same remote archive path. Reserve the stems. const RESERVED_RUNTIME_ASSET_KEYS = new Set(["workspace", "git-workspace"]); // Reject an asset key before any path is built from it. An asset key becomes a // remote directory (`/`), a remote archive name // (`-upload.tar`), and a host temp file (`.tar`). A path separator or // `..` in the key escapes those roots. A reserved stem makes the asset archive // share a path with the workspace archive; under concurrent sync the asset task // and the workspace task then write or upload the same archive at the same time, // which fails extraction nondeterministically or puts asset bytes in the // workspace. Fail closed on both cases. function assertRuntimeAssetKeyIsSafe(key: string): void { if (key.length === 0 || key.includes("/") || key.includes("\\") || key.includes("..")) { throw new Error(`sandbox runtime asset key is not a simple path segment: ${key}`); } if (RESERVED_RUNTIME_ASSET_KEYS.has(key)) { throw new Error(`sandbox runtime asset key collides with a reserved runtime archive name: ${key}`); } } export function parseSandboxRemoteExecutionSpec(value: unknown): SandboxRemoteExecutionSpec | null { const parsed = asObject(value); const transport = asString(parsed.transport).trim(); const provider = asString(parsed.provider).trim(); const sandboxId = asString(parsed.sandboxId).trim(); const remoteCwd = asString(parsed.remoteCwd).trim(); const timeoutMs = asNumber(parsed.timeoutMs); if ( transport !== "sandbox" || provider.length === 0 || sandboxId.length === 0 || remoteCwd.length === 0 || !Number.isFinite(timeoutMs) || timeoutMs <= 0 ) { return null; } return { transport: "sandbox", provider, sandboxId, remoteCwd, timeoutMs, apiKey: asString(parsed.apiKey).trim() || null, }; } export function buildSandboxExecutionSessionIdentity(spec: SandboxRemoteExecutionSpec | null) { if (!spec) return null; return { transport: "sandbox", provider: spec.provider, sandboxId: spec.sandboxId, remoteCwd: spec.remoteCwd, } as const; } export function sandboxExecutionSessionMatches(saved: unknown, current: SandboxRemoteExecutionSpec | null): boolean { const currentIdentity = buildSandboxExecutionSessionIdentity(current); if (!currentIdentity) return false; const parsedSaved = asObject(saved); return ( asString(parsedSaved.transport) === currentIdentity.transport && asString(parsedSaved.provider) === currentIdentity.provider && asString(parsedSaved.sandboxId) === currentIdentity.sandboxId && asString(parsedSaved.remoteCwd) === currentIdentity.remoteCwd ); } async function withTempDir(prefix: string, fn: (dir: string) => Promise): Promise { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); try { return await fn(dir); } finally { await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); } } async function sha256File(filePath: string): Promise { return await new Promise((resolveDigest, rejectDigest) => { const digest = createHash("sha256"); const stream = createReadStream(filePath); stream.on("data", (chunk) => digest.update(chunk)); stream.on("error", rejectDigest); stream.on("end", () => resolveDigest(digest.digest("hex"))); }); } async function copyDurableSeedArchive(input: { sourcePath: string; targetPath: string; expectedSha256?: string | null; }): Promise { const source = await fs.lstat(input.sourcePath); if (source.isSymbolicLink() || !source.isFile()) { throw new Error("workspace_durable_seed_invalid"); } if ( input.expectedSha256 && (await sha256File(input.sourcePath)) !== input.expectedSha256 ) { throw new Error("workspace_durable_seed_digest_mismatch"); } await fs.copyFile(input.sourcePath, input.targetPath, fsConstants.COPYFILE_FICLONE); } async function persistDurableSeedArchive(input: { sourcePath: string; targetPath: string; }): Promise { const parent = path.dirname(input.targetPath); await fs.mkdir(parent, { recursive: true, mode: 0o700 }); const parentStat = await fs.lstat(parent); if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) { throw new Error("workspace_durable_seed_root_invalid"); } const temporary = path.join( parent, `.${path.basename(input.targetPath)}.${randomUUID()}.tmp`, ); try { await fs.copyFile(input.sourcePath, temporary, fsConstants.COPYFILE_FICLONE); await fs.chmod(temporary, 0o600); await fs.rename(temporary, input.targetPath); } finally { await fs.rm(temporary, { force: true }).catch(() => undefined); } } async function execTar(args: string[]): Promise { await execFile("tar", args, { env: { ...process.env, COPYFILE_DISABLE: "1", }, maxBuffer: 32 * 1024 * 1024, }); } export async function createTarballFromDirectory(input: { localDir: string; archivePath: string; exclude?: string[]; followSymlinks?: boolean; }): Promise { const excludeArgs = ["._*", ...(input.exclude ?? [])].flatMap((entry) => ["--exclude", entry]); // Archive the directory's top-level entries BY NAME rather than ".". Archiving // "." embeds a "./" self-entry whose mode/mtime tar then tries to restore onto // the extraction target directory; that chmod/utime fails with "Operation not // permitted" when the target is a directory the extracting (non-root) user does // not own, e.g. an emptyDir mount in a hardened/gVisor sandbox pod. Enumerating // entries avoids the self-entry entirely and is portable across GNU/BSD/busybox // tar (no GNU-only --no-overwrite-dir needed). --exclude still filters nested // matches and any named entry it matches. await withTempDir("paperclip-tar-list-", async (directory) => { const list = path.join(directory, "entries.nul"); const file = await fs.open(list, "wx", 0o600); try { for await (const entry of await fs.opendir(input.localDir)) await file.writeFile(`${entry.name}\0`); } finally { await file.close(); } await execTar(["-c", "--no-xattrs", ...(input.followSymlinks ? ["-h"] : []), "-f", input.archivePath, "-C", input.localDir, ...excludeArgs, "--null", "-T", list]); }); } async function extractTarballToDirectory(input: { archivePath: string; localDir: string; exclude?: string[]; }): Promise { await fs.mkdir(input.localDir, { recursive: true }); await execTar([...(input.exclude ?? []).map(entry => `--exclude=${escapeTarExcludeLiteral(entry)}`), "-xf", input.archivePath, "-C", input.localDir]); } async function* walkDirectory(root: string, relative = "", childrenFirst = false): AsyncGenerator { for await (const entry of await fs.opendir(path.join(root, relative))) { const nextRelative = relative ? path.posix.join(relative, entry.name) : entry.name; if (!childrenFirst) yield nextRelative; if (entry.isDirectory()) yield* walkDirectory(root, nextRelative, childrenFirst); if (childrenFirst) yield nextRelative; } } async function confineFullWorkspaceExport(root: string, rejectExternalSymlinks = false): Promise { const unsafe = () => Object.assign(new Error("Unsafe workspace export directory"), { code: "WORKSPACE_RESTORE_UNSAFE_ARCHIVE" }); if (!(await fs.lstat(root)).isDirectory()) throw unsafe(); const canonicalRoot = await fs.realpath(root); const walk = async (directory: string): Promise => { if (await fs.realpath(directory) !== directory) throw unsafe(); for await (const entry of await fs.opendir(directory)) { const entryPath = path.join(directory, entry.name); const stats = await fs.lstat(entryPath); if (stats.isDirectory()) await walk(entryPath); else if (stats.isSymbolicLink()) { const target = await fs.readlink(entryPath); const resolved = path.resolve(directory, target); if (path.isAbsolute(target) || (resolved !== canonicalRoot && !resolved.startsWith(`${canonicalRoot}${path.sep}`))) { if (rejectExternalSymlinks) throw unsafe(); await fs.unlink(entryPath); } if (rejectExternalSymlinks) { const canonicalTarget = await fs.realpath(entryPath).catch(() => { throw unsafe(); }); if (canonicalTarget !== canonicalRoot && !canonicalTarget.startsWith(`${canonicalRoot}${path.sep}`)) throw unsafe(); } } else if (!stats.isFile()) throw unsafe(); } }; await walk(canonicalRoot); } /** Export into a caller-owned empty staging directory, without a merge baseline * or any writes to the original host workspace. The normal transfer client owns * archive validation; snapshots additionally reject links that cannot be kept. */ export async function exportSandboxWorkspaceSnapshot(input: { client: SandboxManagedRuntimeClient; remoteDir: string; stagingDir: string; operationId: string; }): Promise { await fs.mkdir(input.stagingDir, { recursive: true }); if ((await fs.readdir(input.stagingDir)).length) throw new Error("Workspace snapshot destination is not empty"); const operations: SandboxSyncOperation[] = [{ operationId: input.operationId, files: [{ sourcePath: input.remoteDir, targetPath: input.stagingDir, kind: "directory", exclude: [".paperclip-runtime"] }], }]; assertSyncOperationsConfined(operations, { sourceRoots: [input.remoteDir], targetRoots: [input.stagingDir] }); if (!input.client.syncOut) throw Object.assign(new Error("Validated workspace export is unavailable for this provider"), { code: "WORKSPACE_SNAPSHOT_EXPORT_UNSUPPORTED" }); await input.client.syncOut(operations); await confineFullWorkspaceExport(input.stagingDir, true); } /** Only host-created snapshot archives qualify: the expected digest comes from * a verified preservation receipt, never an agent or upload request. */ export async function restoreVerifiedSnapshotArchive(input: { archivePath: string; localDir: string; sha256: string; byteSize: number; }): Promise { const hash = createHash("sha256"); let size = 0; for await (const bytes of createReadStream(input.archivePath)) { size += bytes.length; if (size > input.byteSize) throw new Error("Workspace snapshot size changed"); hash.update(bytes); } if (size !== input.byteSize || hash.digest("hex") !== input.sha256) throw new Error("Workspace snapshot digest changed"); await fs.mkdir(input.localDir, { recursive: true }); if ((await fs.readdir(input.localDir)).length) throw new Error("Workspace snapshot destination is not empty"); await extractTarballToDirectory(input); await confineFullWorkspaceExport(input.localDir, true); } async function copyWorkspaceEntry(sourceRoot: string, targetRoot: string, relative: string): Promise { const sourcePath = path.join(sourceRoot, relative); const targetPath = path.join(targetRoot, relative); const stats = await fs.lstat(sourcePath); if (stats.isDirectory()) { await fs.mkdir(targetPath, { recursive: true }); return; } await fs.mkdir(path.dirname(targetPath), { recursive: true }); await fs.rm(targetPath, { recursive: true, force: true }).catch(() => undefined); if (stats.isSymbolicLink()) { const linkTarget = await fs.readlink(sourcePath); await fs.symlink(linkTarget, targetPath); return; } const stagedTargetPath = buildUniqueStagingPath({ targetPath, suffix: ".paperclip-copy" }); await fs.rm(stagedTargetPath, { recursive: true, force: true }).catch(() => undefined); try { await fs.copyFile(sourcePath, stagedTargetPath, fsConstants.COPYFILE_FICLONE).catch(async () => { await fs.copyFile(sourcePath, stagedTargetPath); }); await fs.chmod(stagedTargetPath, stats.mode); await fs.rename(stagedTargetPath, targetPath); } finally { await fs.rm(stagedTargetPath, { recursive: true, force: true }).catch(() => undefined); } } export async function mirrorDirectory( sourceDir: string, targetDir: string, options: { preserveAbsent?: string[] } = {}, ): Promise { await fs.mkdir(targetDir, { recursive: true }); const preserveAbsent = new Set(options.preserveAbsent ?? []); const shouldPreserveAbsent = (relative: string) => [...preserveAbsent].some((candidate) => isRelativePathOrDescendant(relative, candidate)); for await (const relative of walkDirectory(targetDir, "", true)) { if (shouldPreserveAbsent(relative)) continue; const sourceExists = await fs.lstat(path.join(sourceDir, relative)).then(() => true).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT" || error.code === "ENOTDIR") return false; throw error; }); if (!sourceExists) { await fs.rm(path.join(targetDir, relative), { recursive: true, force: true }).catch(() => undefined); } } for await (const relative of walkDirectory(sourceDir)) { await copyWorkspaceEntry(sourceDir, targetDir, relative); } } interface WorkspaceSourceRoot { sourceDir: string; dev: number; ino: number; } async function captureWorkspaceSourceRoot(localDir: string): Promise { const sourceDir = await fs.realpath(localDir); const stats = await fs.lstat(sourceDir); if (!stats.isDirectory()) throw new Error("Workspace overlay root is not a directory"); return { sourceDir, dev: stats.dev, ino: stats.ino }; } async function copySelectedWorkspaceEntries(input: { sourceRoot: WorkspaceSourceRoot; targetDir: string; relativePaths: WorkspacePaths; ignoredPaths?: WorkspacePaths; exclude: string[]; }): Promise { await fs.mkdir(input.targetDir, { recursive: true }); const { sourceDir, dev, ino } = input.sourceRoot; const assertSourceRoot = async () => { const current = await fs.lstat(sourceDir); if (!current.isDirectory() || current.dev !== dev || current.ino !== ino) { throw new Error("Workspace overlay root directory changed during staging"); } }; await assertSourceRoot(); const ignored = workspacePathMatcher(input.ignoredPaths); try { for (const relative of workspacePaths(input.relativePaths)) { if (shouldExcludePath(relative, input.exclude) || ignored.matches(relative)) continue; const sourcePath = path.join(sourceDir, relative); const parentSegments = path.relative(sourceDir, path.dirname(sourcePath)).split(path.sep).filter(Boolean); const assertParentDirectory = async () => { // Git selected this path before staging. A replaced ancestor must not // redirect the copy through a symlink, even to another workspace folder. // Inspect types instead of comparing realpath spelling: case-insensitive // filesystems can resolve Git's indexed casing to a renamed directory. let parentPath = sourceDir; for (const segment of parentSegments) { if (segment === "..") throw new Error(`Workspace overlay directory escapes its root: ${relative}`); parentPath = path.join(parentPath, segment); if (!(await fs.lstat(parentPath)).isDirectory()) { throw new Error(`Workspace overlay ancestor is not a directory: ${relative}`); } } }; // Include root-level entries, and do not treat a missing root as an // ordinary source file that disappeared after the snapshot. await assertSourceRoot(); try { await assertParentDirectory(); await fs.lstat(sourcePath); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; throw error; } await copyWorkspaceEntry(sourceDir, input.targetDir, relative); // Do not upload the staged tree if an ancestor changed during the copy. await assertSourceRoot(); await assertParentDirectory(); } } finally { ignored.close(); } } function toBuffer(bytes: Buffer | Uint8Array | ArrayBuffer): Buffer { if (Buffer.isBuffer(bytes)) return bytes; if (bytes instanceof ArrayBuffer) return Buffer.from(bytes); return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); } // Sum `bytesTransferred` over a `SandboxSyncResult`. The result crosses the // plugin boundary, so a provider can return anything: guard each value and // treat a missing, non-finite, or negative number as 0. function sumSyncResultBytes(result: SandboxSyncResult): number { return result.operations.reduce((total, operation) => { const bytes = operation.bytesTransferred; return Number.isFinite(bytes) && bytes > 0 ? total + bytes : total; }, 0); } function tarExcludeFlags(exclude: string[] | undefined): string { return ["._*", ...(exclude ?? [])].map((entry) => `--exclude ${shellQuote(entry)}`).join(" "); } function createRemoteTarballFromDirectoryCommand(input: { remoteDir: string; archivePath: string; exclude?: string[]; }): string { const list = `${input.archivePath}.entries`; return [ `mkdir -p ${shellQuote(path.posix.dirname(input.archivePath))}`, `cd ${shellQuote(input.remoteDir)}`, `find . -mindepth 1 -maxdepth 1 -print0 > ${shellQuote(list)}`, `tar -cf ${shellQuote(input.archivePath)} ${tarExcludeFlags(input.exclude)} --null -T ${shellQuote(list)}`, `rm -f -- ${shellQuote(list)}`, ].join(" && "); } async function emitRuntimeStatus( sink: RuntimeStatusSink | undefined, phase: RuntimeStatusPhase, message: string, ): Promise { if (!sink) return; await Promise.resolve(sink({ phase, message })).catch(() => undefined); } export function mergeExcludes(...groups: Array): string[] { return [...new Set(groups.flatMap((group) => group ?? []))]; } function preserveFindArgs(entries: string[]): string { return entries.map((entry) => `! -name ${shellQuote(entry)}`).join(" "); } // Bridge a single byte-level transfer to the throttled progress reporter. The // transport reports decoded bytes via `options.onProgress`; the reporter turns // them into a throttled, fully-formatted log line. `finish()` emits the terminal // completion line (idempotent) once the transfer returns. function makeTransferProgress( sink: RuntimeProgressSink | undefined, phase: RuntimeProgressPhase, direction: RuntimeProgressDirection, label?: string, runtimeStatus?: { sink: RuntimeStatusSink | undefined; phase: RuntimeStatusPhase; }, ): { options: SandboxTransferProgressOptions | undefined; finish: (doneBytes?: number, totalBytes?: number | null) => Promise; } { if (!sink && !runtimeStatus?.sink) { return { options: undefined, finish: async () => {} }; } const reporter = createRuntimeProgressReporter({ sink: async (line) => { await sink?.(line); if (runtimeStatus?.sink) { await emitRuntimeStatus( runtimeStatus.sink, runtimeStatus.phase, line.replace(/^\[paperclip\]\s*/, "").trim(), ); } }, phase, direction, target: "sandbox", label, }); return { options: { onProgress: async (transferredBytes, totalBytes) => { await reporter.report(transferredBytes, totalBytes); }, }, finish: async (doneBytes, totalBytes) => { await reporter.complete(doneBytes, totalBytes); }, }; } export async function prepareSandboxManagedRuntime(input: { spec: SandboxRemoteExecutionSpec; adapterKey: string; client: SandboxManagedRuntimeClient; workspaceLocalDir: string; workspaceRemoteDir?: string; /** Host-owned transfer scratch within the lease's reserved runtime tree. * Persistent subdirectories can keep scratch outside their native file root. */ runtimeRootDir?: string; syncWorkspace?: boolean; /** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */ workspaceInboundMode?: WorkspaceInboundMode; workspaceSnapshotIsolation?: "worker"; /** Fully await a restored Git index refresh inside the existing inbound barrier. */ workspaceGitIndexRefresh?: boolean; workspaceDurableSeed?: WorkspaceDurableSeedPaths; workspaceCheckpoint?: boolean; onWorkspaceCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; workspaceSeedCacheDirectory?: string; workspaceSeedCacheCompanyDirectory?: string; workspaceSeedCacheScope?: "workspace" | "company"; /** Durable snapshots supplied when reconstructing an interrupted restore. */ workspaceBaseline?: DirectorySnapshot; workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceRepositories?: NonNullable; workspaceExclude?: string[]; workspaceFileMode?: "all"; preserveAbsentOnRestore?: string[]; assets?: SandboxManagedRuntimeAsset[]; /** * Referenced (additional) projects to stage into the sandbox as plain, * read-only trees, each in its own isolated `project-` directory. * Defaults to none, so a legacy/anchor-only call is behavior-identical. */ additionalSources?: SandboxAdditionalSource[]; // Upload progress sink. Threaded for the byte-counting transport rewrite; the // child task wires it into writeFile/readFile. onProgress?: RuntimeProgressSink; onRuntimeProgress?: RuntimeStatusSink; // Optional host span runner for the workspace tarball build. When present, the // host builds both workspace tarballs inside one span named `pack`, so the // host pack time is visible under the `stage.sync` step. The default is a // no-op that keeps the current behavior and control flow. A throwing runner // never changes control flow (see `createRuntimeSpanRunner`). runtimeSpan?: RuntimeSpanRunner; /** Backoff before each automatic workspace copy-back retry. Tests shorten it. */ workspaceRestoreRetryDelaysMs?: readonly number[]; }): Promise { const captureWorkspaceSnapshot = input.workspaceSnapshotIsolation === "worker" ? captureDirectorySnapshotOffThread : captureDirectorySnapshot; const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd; const runtimeRootDir = input.runtimeRootDir ?? path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey); if (input.runtimeRootDir !== undefined) { const reservedRoot = path.posix.join(input.spec.remoteCwd, ".paperclip-runtime"); if (!path.posix.isAbsolute(runtimeRootDir) || path.posix.normalize(runtimeRootDir) !== runtimeRootDir || runtimeRootDir.includes("\0") || runtimeRootDir.endsWith("/") || !runtimeRootDir.startsWith(`${reservedRoot}/`)) { throw new Error("Transfer scratch must remain within the lease runtime tree"); } } // A workspace directory that does not exist on this host has nothing to // stage, no files for ignore rules to govern, and nothing to restore into — // callers that only stage credential assets (the adapter env tests) hand // the runtime a fresh path. Treat that one case as "do not sync" rather // than letting the ignore scan or the staging walk die on ENOENT. Any other // access failure still fails the preparation: a workspace that exists but // cannot be read must not silently become an empty remote workspace. const syncWorkspace = input.syncWorkspace !== false && (await fs.access(input.workspaceLocalDir).then( () => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; }, )); const workspaceInboundMode = input.workspaceInboundMode ?? "host_current"; const stageWorkspace = syncWorkspace && workspaceInboundMode !== "adopt_remote"; const prepareWorkspaceSeed = syncWorkspace && workspaceInboundMode === "adopt_remote" && input.workspaceDurableSeed !== undefined; if (workspaceInboundMode === "durable_seed" && !input.workspaceDurableSeed) { throw new Error("workspace_durable_seed_missing"); } // Reject any unsafe asset key before an archive path or an asset directory is // built from it. This runs before the git snapshot work so a bad key fails fast. for (const asset of input.assets ?? []) { assertRuntimeAssetKeyIsSafe(asset.key); } // Wrap a host-side staging sub-step or one scheduler task in its own span when // the caller injects a runtime span runner. The runner defaults to a no-op, so // a caller with no injected runner keeps the current control flow, and a // throwing runner never changes it (see `createRuntimeSpanRunner`). The two // pre-`pack` operations — the git enumeration and the baseline content-hash // walk — parent under the `stage.sync` step, so they stop showing up as a // hidden gap at the head of the step. Each inbound task (`stage.workspace`, // `stage.asset.`, `stage.project.`) and each outbound restore task // (`restore.workspace`, `restore.asset.`) opens its own span, so two // concurrent tasks produce overlapping spans and the `pack` span nests under // `stage.workspace`. The outbound spans parent under the run's `sandbox.syncBack` // span, because the teardown runs the restore inside that span. const runStepSpan = (name: string, work: () => Promise): Promise => input.runtimeSpan ? input.runtimeSpan(name, work) : work(); const reportWorkspaceStatus = (phase: RuntimeStatusPhase, message: string) => runStepSpan("workspace.progress", () => emitRuntimeStatus(input.onRuntimeProgress, phase, message)); // Resolve an existing workspace alias once, before reading its snapshot. // All subsequent work uses that root, so retargeting the alias cannot select // another repository. Staging also verifies the captured directory identity. const workspaceRoot = syncWorkspace ? await runStepSpan("workspace.source_root", () => captureWorkspaceSourceRoot(input.workspaceLocalDir)) : null; if (workspaceRoot) input = { ...input, workspaceLocalDir: workspaceRoot.sourceDir }; // The git enumeration (`git status --ignored`, the HEAD diffs, `ls-files`). // It reads git's own bookkeeping to decide what to include/exclude, so it is // usually fast, but on a large working tree the `--ignored` walk is not free. const gitSnapshot = syncWorkspace && (input.workspaceGitSnapshot !== undefined || input.workspaceFileMode !== "all") ? input.workspaceGitSnapshot !== undefined ? input.workspaceGitSnapshot : await runStepSpan("snapshot.git", () => readGitWorkspaceSnapshot(input.workspaceLocalDir), ) : null; const repositories = gitSnapshot ? (gitSnapshot.repositories ?? []) : input.workspaceRepositories ?? (syncWorkspace ? await runStepSpan("workspace.repositories", () => readManagedWorkspaceRepositories(input.workspaceLocalDir)) : []); const repositoryGitPaths = (children: typeof repositories, prefix = ""): string[] => children.flatMap(repo => { const relative = prefix ? `${prefix}/${repo.path}` : repo.path; return [`${relative}/.git`, ...repositoryGitPaths(repo.snapshot.repositories ?? [], relative)]; }); const repositoryMetadataExcludes = repositoryGitPaths(repositories); let managedIgnored: Awaited> | null = null; let ignoredPaths = input.workspaceFileMode === "all" ? undefined : gitSnapshot?.ignoredPaths; if (!gitSnapshot && repositories.length) { managedIgnored = await createWorkspaceManifest(); for (const repo of repositories) { if (input.workspaceFileMode !== "all") for (const relative of workspacePaths(repo.snapshot.ignoredPaths)) managedIgnored.add("ignored", `${repo.path}/${relative}`); for (const relative of workspacePaths(repo.snapshot.deletedPaths)) managedIgnored.add("deleted", `${repo.path}/${relative}`); } ignoredPaths = managedIgnored.paths("ignored"); } let deletedPaths = gitSnapshot?.deletedPaths ?? managedIgnored?.paths("deleted") ?? []; let filesystemDeleted: Awaited> | null = null; managedIgnored?.close(); let baselineSnapshot: DirectorySnapshot | null = null; const cleanupWorkspaceSnapshot = async () => { await disposeDirectorySnapshot(baselineSnapshot); if (input.workspaceBaseline && input.workspaceBaseline !== baselineSnapshot) await disposeDirectorySnapshot(input.workspaceBaseline); await disposeGitWorkspaceSnapshot(gitSnapshot); if (!gitSnapshot) for (const repo of repositories) await disposeGitWorkspaceSnapshot(repo.snapshot); if (managedIgnored) await fs.rm(path.dirname(managedIgnored.filePath), { recursive: true, force: true }); if (filesystemDeleted) await fs.rm(path.dirname(filesystemDeleted.filePath), { recursive: true, force: true }); }; try { // A selected subfolder has no cloneable Git snapshot, but its parent // repository's ignore rules still govern which files may leave the host. // Use the same bounded, path-relative resolver as referenced project trees. const directoryIgnore = syncWorkspace && !gitSnapshot && input.workspaceFileMode !== "all" ? await runStepSpan("workspace.ignore_rules", () => resolveReferencedSourceIgnore(input.workspaceLocalDir)) : null; if (directoryIgnore?.kind === "failed") { throw new Error(`Workspace ignore scan failed: ${directoryIgnore.reason}`); } const gitIgnoredExcludes = directoryIgnore?.kind === "git" ? directoryIgnore.ignoredPaths : undefined; const workspaceArchiveExclude = mergeExcludes( [".paperclip-runtime"], repositoryMetadataExcludes, input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES], input.workspaceExclude, gitIgnoredExcludes, ); const restoreExclude = mergeExcludes( repositoryMetadataExcludes, input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], input.preserveAbsentOnRestore, input.workspaceExclude, gitIgnoredExcludes, ); const workspaceRestoreExclude = mergeExcludes(restoreExclude, repositories.map((repo) => repo.path)); // The baseline "before" snapshot: a recursive walk of the whole workspace that baselineSnapshot = syncWorkspace ? (input.workspaceBaseline ?? (await runStepSpan("snapshot.baseline", () => captureWorkspaceSnapshot(input.workspaceLocalDir, { exclude: restoreExclude, ignoredPaths, diskBacked: true, }), ))) : null; if (input.workspaceBaseline && repositoryMetadataExcludes.length) { baselineSnapshot = await selectDirectorySnapshot(input.workspaceBaseline, { omit: repositoryMetadataExcludes, exclude: mergeExcludes(input.workspaceBaseline.exclude, repositoryMetadataExcludes), ignoredPaths: input.workspaceBaseline.ignoredPaths, }); } if (input.workspaceFileMode === "all" && baselineSnapshot && (isPathManifest(deletedPaths) ? deletedPaths.count : deletedPaths.length) > 0) { filesystemDeleted = await createWorkspaceManifest(); try { for (const relative of workspacePaths(deletedPaths)) { if (!baselineSnapshot.entries.has(relative)) filesystemDeleted.add("deleted", relative); } deletedPaths = filesystemDeleted.paths("deleted"); } finally { filesystemDeleted.close(); } } const seedGeneration = input.workspaceSeedCacheDirectory && baselineSnapshot && workspaceInboundMode !== "durable_seed" ? await runStepSpan("workspace.seed_generation", async () => workspaceSeedGeneration(baselineSnapshot!, gitSnapshot, repositories, input.workspaceSeedCacheScope)) : null; const sharedSeed = input.workspaceSeedCacheScope === "company" && seedGeneration !== null; // Every inbound staging step delegates to the provider through `client.syncIn`: // the orchestrator no longer inlines `writeFile`+`run` or chooses a transport, // and there is no `usesCustomProvision` native-diversion gate. `syncIn` is // ALWAYS present in production — the command-managed client exposes a native // transport (Daytona/Kubernetes `uploadFiles` + provider-executed post-upload // commands) or a byte-identical base64-tar fallback that reproduces the prior // `writeFile`+`run` sequence. Require it explicitly so a misconfigured client // fails loud rather than silently skipping staging. `syncOut` stays optional // (native-only) with a tar fallback on the restore path below. const syncIn = input.client.syncIn; if (typeof syncIn !== "function") { throw new Error( "prepareSandboxManagedRuntime requires a client that exposes syncIn " + "(createCommandManagedRuntimeClient provides a native-or-fallback implementation).", ); } const nativeSyncOut = typeof input.client.syncOut === "function"; let syncOperationSeq = 0; // Opaque, ordered, non-sensitive operation tokens — never a caller/asset id. const nextSyncOperationId = () => `sync-op-${++syncOperationSeq}`; // Remote directory of each additional (referenced) project that stages // successfully, keyed by projectId. A project that fails to stage is absent. const additionalSourceDirs: Record = {}; // Each additional (referenced) project whose staging failed, paired with the // failure message. Per-project failure isolation keeps the run and the other // projects going; this list makes each failure a first-class, reported outcome. const additionalSourceFailures: AdditionalSourceStagingFailure[] = []; // Additional projects stage as plain trees. Drop the heavy build/cache dirs a // reference tree does not need, and `.git` — additional sources never carry // git-history semantics (anchor-only). Each project also drops its OWN // resolved Git-ignored paths (or keeps this fixed set as-is for a non-Git // source) — see `resolveReferencedSourceIgnore` and the per-project merge // below. const additionalSourceBaseExclude = mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [".git"]); // Every delegated post-upload command (extract/wipe/remove-deleted/asset merge) // must run under the run-specific timeout (`spec.timeoutMs`), not the provider // sync client's default timeout — the two can differ, and before staging was // routed through `syncIn` each of these ran via // `client.run(cmd, { timeoutMs: spec.timeoutMs })`. When they mismatch, a // command left without a `timeoutMs` outlives (or is killed under) the wrong // limit. Stamp the run timeout onto every delegated command, preserving any // command that already carries its own explicit timeout. const withRunTimeout = ( commands: SandboxPostUploadCommand[], ): SandboxPostUploadCommand[] => commands.map((command) => ({ ...command, timeoutMs: command.timeoutMs ?? input.spec.timeoutMs, })); await withTempDir("paperclip-sandbox-sync-", async (tempDir) => { const cachedSeed = seedGeneration ? await runStepSpan("workspace.seed_lookup", () => readWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, { companyDirectory: input.workspaceSeedCacheCompanyDirectory, privateCopyDirectory: tempDir, runtimeSpan: runStepSpan })) : null; const replaySeed = workspaceInboundMode === "durable_seed" ? input.workspaceDurableSeed : cachedSeed; const preservedNames = new Set([ ".paperclip-runtime", ...(gitSnapshot ? [".git"] : []), ...(input.preserveAbsentOnRestore ?? []), ]); // Stage one source directory into an isolated remote subdirectory through the // unified `syncIn` seam. Both the workspace/asset tar path and the additional // (referenced) project path use it: build one `SandboxSyncOperation` from the // caller's `files` mappings (a host tarball as a single `file` mapping, or a // whole `directory` mapping), carry any extract/wipe/merge steps as ordered // `postUploadCommands`, confine the operation's source and target to their // own roots (fail-closed), and delegate to `syncIn` (native transfer, or the // base64-tar fallback). `finish` emits the terminal progress line. const stageConfinedSyncIn = async (params: { files: SandboxSyncFileMapping[]; postUploadCommands?: SandboxPostUploadCommand[]; sourceRoots: string[]; targetRoots: string[]; progressLabel: string; statusPhase: RuntimeStatusPhase; progressBytes: number; }): Promise => { const operations: SandboxSyncOperation[] = [{ operationId: nextSyncOperationId(), files: params.files, ...(params.postUploadCommands ? { postUploadCommands: withRunTimeout(params.postUploadCommands) } : {}), }]; assertSyncOperationsConfined(operations, { sourceRoots: params.sourceRoots, targetRoots: params.targetRoots, }); const upload = makeTransferProgress( input.onProgress, "Syncing", "to", params.progressLabel, { sink: input.onRuntimeProgress, phase: params.statusPhase }, ); const syncResult = await runStepSpan("workspace.transfer", () => syncIn(operations)); // Prefer the transport's own byte total. It is the real count for a // provider that has no host tarball to stat (a referenced project rides // a `directory` mapping). Fall back to the caller-supplied count when // the transport reports 0, so a provider that under-reports still // shows the host-known workspace total. const transferredBytes = sumSyncResultBytes(syncResult); const reportedBytes = transferredBytes > 0 ? transferredBytes : params.progressBytes; await runStepSpan("workspace.progress", () => upload.finish(reportedBytes, reportedBytes)); }; // Build the ordered inbound operation task list. Each task stages one inbound // operation from start to end: it packs the bytes, confines the mappings, and // uploads them. The shared scheduler starts the tasks. When the sync client // permits concurrency, the scheduler keeps at most SYNC_OPERATION_CONCURRENCY_LIMIT // tasks active. When the client does not permit concurrency, the scheduler runs // one task at a time in this order. The scheduler settles every started task // before it returns, so no upload outlives the coordinator and ACP never starts // before the barrier. const inboundTasks: Array> = []; // A required-flag list, one entry per task and index-aligned with inboundTasks. // The workspace and each asset are required operations: a rejection is fatal // after the barrier. Each referenced project is a nonfatal operation: its task // records its own failure and never rejects. const inboundTaskIsRequired: boolean[] = []; if (stageWorkspace || prepareWorkspaceSeed) { inboundTaskIsRequired.push(true); inboundTasks.push(() => runStepSpan("stage.workspace", async () => { // A git-backed workspace and a plain workspace both stage through ONE // confined `syncIn` operation. A git-backed workspace carries TWO host tars — // the git-history clone and the working-tree overlay — as two `file` mappings // on the SAME operation, with their extract commands as ordered // `postUploadCommands`. One operation shares one mkdir, one confine guard, one // `uploadFiles`, and one rename exec, so the second `syncIn` round trip is // removed. Build the whole merged file set and command list BEFORE the confine // guard runs (inside `stageConfinedSyncIn`); never append a mapping after it. const workspaceFiles: SandboxSyncFileMapping[] = []; const workspacePostUploadCommands: SandboxPostUploadCommand[] = []; let workspaceUploadBytes = 0; const gitSeedTarPath = cachedSeed?.gitArchivePath ?? path.join(tempDir, sharedSeed ? "git-seed.tar" : "git-workspace.tar"); // Build both host tarballs (the git-history tar and the workspace-overlay // tar) inside one host span named `pack`. This span makes the host pack // time visible under the `stage.sync` step, where it is otherwise a hidden // gap with no span. The transfer (`stageConfinedSyncIn`) runs after this // span, so `pack` measures only the host tar-build cost. The runner // defaults to a no-op, so a caller with no injected runner keeps the // current behavior and control flow. await runStepSpan("pack", async () => { // 1. git-history tar (git-backed workspace only). Both tar targets live under // `runtimeRootDir` (`.paperclip-runtime/`). The git extract // wipes the target tree EXCEPT `.paperclip-runtime`, so the overlay tar, // which sits under `.paperclip-runtime`, survives to run its own extract. if (gitSnapshot || repositories.length) { await reportWorkspaceStatus("git_sync", "Syncing git history to environment"); const gitTarPath = sharedSeed ? path.join(tempDir, "git-workspace.tar") : gitSeedTarPath; const remoteGitTar = path.posix.join( runtimeRootDir, "git-workspace-upload.tar", ); if (replaySeed) { const durableGitArchive = replaySeed.gitArchivePath; if (!durableGitArchive) { throw new Error("workspace_durable_seed_git_missing"); } if (!cachedSeed) await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ sourcePath: durableGitArchive, targetPath: gitSeedTarPath, expectedSha256: replaySeed.gitArchiveSha256, })); } else { const archive = async (cloneDir: string) => runStepSpan("workspace.git_archive", () => createTarballFromDirectory({ localDir: cloneDir, archivePath: gitSeedTarPath, exclude: [".paperclip-runtime"] })); const snapshot = (git: GitWorkspaceSnapshot) => sharedSeed ? workspaceSeedCloneSnapshot(git) : git; if (gitSnapshot) await runStepSpan("workspace.git_clone", () => withShallowGitWorkspaceClone({ localDir: input.workspaceLocalDir, snapshot: snapshot(gitSnapshot) }, archive)); else await runStepSpan("workspace.git_clone", () => withTempDir("paperclip-managed-repositories-", async (cloneDir) => { for (const repo of repositories) await withShallowGitWorkspaceClone({ localDir: path.join(input.workspaceLocalDir, repo.path), snapshot: snapshot(repo.snapshot) }, async (child) => copyCloneTree(child, path.join(cloneDir, repo.path))); await archive(cloneDir); })); } if (sharedSeed) await bindWorkspaceSeedGitArchive({ sourcePath: gitSeedTarPath, targetPath: gitTarPath, directory: path.join(tempDir, "git-binding"), git: gitSnapshot, repositories, }); if ((!replaySeed || cachedSeed) && input.workspaceDurableSeed?.gitArchivePath) { await runStepSpan("workspace.seed_persist", () => persistDurableSeedArchive({ sourcePath: gitTarPath, targetPath: input.workspaceDurableSeed!.gitArchivePath!, })); } workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir, }); workspacePostUploadCommands.push({ command: buildWorkspaceTarExtractCommand({ workspaceRemoteDir, remoteTar: remoteGitTar, wipeExceptNames: [".paperclip-runtime"], }), }); workspaceUploadBytes += (await fs.stat(gitTarPath)).size; } // 2. workspace-overlay tar. A git-backed overlay merges on top of the just // extracted git tree (no wipe); a plain workspace wipes every child except // the preserved names first. The extract runs AFTER the git extract. await reportWorkspaceStatus("config_sync", "Syncing workspace to environment"); const workspaceTarPath = cachedSeed?.workspaceArchivePath ?? path.join(tempDir, "workspace.tar"); if (replaySeed) { if (!cachedSeed) await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ sourcePath: replaySeed.workspaceArchivePath, targetPath: workspaceTarPath, expectedSha256: replaySeed.workspaceArchiveSha256, })); } else { const workspaceArchiveDir = gitSnapshot || repositories.length ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir; if (gitSnapshot || repositories.length) { await runStepSpan("workspace.overlay_copy", () => copySelectedWorkspaceEntries({ sourceRoot: workspaceRoot!, targetDir: workspaceArchiveDir, relativePaths: (input.workspaceFileMode === "all" ? undefined : gitSnapshot?.overlayPaths) ?? (baselineSnapshot!.entries instanceof WorkspaceManifestMap ? baselineSnapshot!.entries.manifest : [...baselineSnapshot!.entries.keys()]), ignoredPaths, exclude: workspaceArchiveExclude, })); } await runStepSpan("workspace.overlay_archive", () => createTarballFromDirectory({ localDir: workspaceArchiveDir, archivePath: workspaceTarPath, exclude: gitSnapshot || repositories.length ? undefined : workspaceArchiveExclude, })); if (input.workspaceDurableSeed) { await runStepSpan("workspace.seed_persist", () => persistDurableSeedArchive({ sourcePath: workspaceTarPath, targetPath: input.workspaceDurableSeed!.workspaceArchivePath, })); } } if (cachedSeed && input.workspaceDurableSeed) await runStepSpan("workspace.seed_persist", () => persistDurableSeedArchive({ sourcePath: workspaceTarPath, targetPath: input.workspaceDurableSeed!.workspaceArchivePath })); const remoteWorkspaceTar = path.posix.join( runtimeRootDir, "workspace-upload.tar", ); workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir, }); workspacePostUploadCommands.push({ command: buildWorkspaceTarExtractCommand({ workspaceRemoteDir, remoteTar: remoteWorkspaceTar, wipeExceptNames: gitSnapshot || repositories.length ? null : [...preservedNames], exclude: repositoryMetadataExcludes, }), }); // 3. Optional remove-deleted-paths command runs LAST, after both extracts. if ((isPathManifest(deletedPaths) ? deletedPaths.count : deletedPaths.length) > 0) { const deletedManifest = path.join(tempDir, "deleted.nul"); const remoteDeletedManifest = path.posix.join(runtimeRootDir, "deleted.nul"); await runStepSpan("workspace.deleted_manifest", () => writeWorkspacePaths(deletedPaths, deletedManifest)); workspaceFiles.push({ sourcePath: deletedManifest, targetPath: remoteDeletedManifest, kind: "file", access: "rw", writablePath: runtimeRootDir }); workspacePostUploadCommands.push({ command: buildRemoveDeletedPathsCommand({ remoteDir: workspaceRemoteDir, manifestPath: remoteDeletedManifest }) }); } workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size; }); const publishSeed = async () => { if (seedGeneration && !cachedSeed && input.workspaceDurableSeed) { const seed = input.workspaceDurableSeed; const gitArchivePath = gitSnapshot || repositories.length ? (sharedSeed ? gitSeedTarPath : seed.gitArchivePath) : null; await runStepSpan("workspace.seed_publish", () => publishWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, { ...seed, gitArchivePath: gitSnapshot || repositories.length ? seed.gitArchivePath : null }, { companyDirectory: input.workspaceSeedCacheCompanyDirectory, scope: input.workspaceSeedCacheScope, ...(sharedSeed && gitArchivePath ? { cacheOnlyGitArchivePath: gitArchivePath } : {}), runtimeSpan: runStepSpan, verify: () => runStepSpan("seed.verify", async () => { const verifiedRoot = path.join(tempDir, "verify-seed"); try { if (gitSnapshot || repositories.length) await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: gitSeedTarPath, localDir: verifiedRoot })); await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: path.join(tempDir, "workspace.tar"), localDir: verifiedRoot, exclude: repositoryMetadataExcludes })); for (const relative of workspacePaths(deletedPaths)) { let parent = verifiedRoot; for (const segment of relative.split("/").slice(0, -1)) { parent = path.join(parent, segment); const stat = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); if (stat && !stat.isDirectory()) return false; } await fs.rm(path.join(verifiedRoot, relative), { recursive: true, force: true }); } const verified = await runStepSpan("workspace.seed_snapshot", () => captureWorkspaceSnapshot(verifiedRoot, { exclude: baselineSnapshot!.exclude, ignoredPaths: baselineSnapshot!.ignoredPaths, diskBacked: true })); try { return directorySnapshotSha256(verified) === directorySnapshotSha256(baselineSnapshot!); } finally { await disposeDirectorySnapshot(verified); } } finally { await runStepSpan("workspace.seed_cleanup", () => fs.rm(verifiedRoot, { recursive: true, force: true })); } }), })); } }; if (!stageWorkspace) { await publishSeed(); return; } // One confined `syncIn` for the whole merged workspace file set. The confine // guard covers every mapping BEFORE any bytes upload (fail-closed): a source // or target escape in EITHER tar mapping stops the upload of both. const staging = stageConfinedSyncIn({ files: workspaceFiles, postUploadCommands: workspacePostUploadCommands, sourceRoots: [tempDir], targetRoots: [runtimeRootDir], progressLabel: "workspace", statusPhase: "config_sync", progressBytes: workspaceUploadBytes, }).then(async () => { if (input.workspaceGitIndexRefresh && gitSnapshot && workspaceInboundMode === "host_current") { await runStepSpan("workspace.git_index_refresh", () => input.client.run( buildRefreshWorkspaceGitIndexCommand(workspaceRemoteDir), { timeoutMs: input.spec.timeoutMs }, )); } }); const [staged] = await Promise.allSettled([staging, publishSeed()]); if (staged.status === "rejected") throw staged.reason; }), ); } for (const asset of input.assets ?? []) { inboundTaskIsRequired.push(true); inboundTasks.push(() => runStepSpan(`stage.asset.${asset.key}`, async () => { await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing runtime assets to environment"); const remoteAssetDir = path.posix.join(runtimeRootDir, asset.key); const remoteAssetTar = path.posix.join(runtimeRootDir, `${asset.key}-upload.tar`); // Every asset — default OR custom-provisioned (e.g. an adapter credential // merge) — rides one `syncIn` operation: the asset tar plus any staged // helper files as `files` mappings, and the extract/merge command as the // ordered post-upload command. There is no native-diversion gate; a // custom-provisioned asset's bytes now ride native `uploadFiles` and its // command runs as a provider-executed post-upload command. const assetTarPath = path.join(tempDir, `${asset.key}.tar`); await createTarballFromDirectory({ localDir: asset.localDir, archivePath: assetTarPath, followSymlinks: asset.followSymlinks, exclude: asset.exclude, }); const files: SandboxSyncFileMapping[] = [ { sourcePath: assetTarPath, targetPath: remoteAssetTar, kind: "file", access: "rw", writablePath: remoteAssetDir }, ]; // Stage provision helper files (e.g. the merge scripts) into the temp dir // and map them alongside the asset tar so they ride the same native upload. for (const stageFile of asset.provision?.stageFiles ?? []) { const safeName = stageFile.name; if (/[\\/]|\.\.(\.|$)/.test(safeName) || safeName === "..") { throw new Error(`provision stageFile.name must be a simple basename, got: ${safeName}`); } const stageBytes = typeof stageFile.contents === "string" ? Buffer.from(stageFile.contents) : stageFile.contents; const stageHostPath = path.join(tempDir, `${asset.key}.stage.${safeName}`); await fs.writeFile(stageHostPath, stageBytes); // A stage helper file (for example a merge script) is a read-only input // that the provision command reads; the agent does not change it and does // not keep it. So it is `access: "ro"` and never joins the writable set. files.push({ sourcePath: stageHostPath, targetPath: path.posix.join(runtimeRootDir, safeName), kind: "file", access: "ro", }); } const postUploadCommand = asset.provision?.postUploadCommand?.({ assetTarPath: remoteAssetTar, assetDir: remoteAssetDir, runtimeRootDir, }) ?? buildDefaultExtractRuntimeAssetCommand({ remoteAssetDir, remoteAssetTar }); const assetTarSize = (await fs.stat(assetTarPath)).size; await stageConfinedSyncIn({ files, postUploadCommands: [{ command: postUploadCommand }], sourceRoots: [tempDir], targetRoots: [runtimeRootDir], progressLabel: asset.key, statusPhase: "config_sync", progressBytes: assetTarSize, }); }), ); } // Stage each referenced (additional) project as a plain, read-only tree in // its OWN isolated remote directory (`project-`). An additional // project rides one confined `syncIn` directory mapping — a native directory // transfer, or the base64-tar fallback — with source and target confined to // their own roots. No workspace, git-history, or `.paperclip-runtime` // semantics apply; those stay anchor-only. Per-project failure isolation: one // project's confinement or sync failure logs a warning and is skipped, and // the run plus the other projects continue. Only a project that stages // successfully appears in `additionalSourceDirs`. const additionalSourceList = input.additionalSources ?? []; // Record each referenced-project failure in its input-order slot. Slot order // keeps the failure list stable no matter the task completion order under // concurrency. const additionalFailureSlots: Array = additionalSourceList.map(() => null); additionalSourceList.forEach((source, sourceIndex) => { inboundTaskIsRequired.push(false); inboundTasks.push(() => runStepSpan(`stage.project.${source.projectId}`, async () => { const { localPath, projectId, ignoreResolution } = source; const label = `project-${projectId}`; try { if (!path.posix.isAbsolute(localPath)) { throw new Error(`additional source localPath is not an absolute path: ${localPath}`); } if ( projectId.length === 0 || projectId.includes("/") || projectId.includes("\\") || projectId.includes("..") ) { throw new Error(`additional source projectId is not a simple path segment: ${projectId}`); } // Fail closed: a project whose ignore resolution failed is not staged // at all. Shipping it with only the fixed heavy-directory excludes // would defeat the resolution's purpose. if (ignoreResolution.kind === "failed") { throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`); } const remoteProjectDir = path.posix.join(runtimeRootDir, label); const exclude = mergeExcludes( additionalSourceBaseExclude, referencedSourceIgnoreExcludeEntries(ignoreResolution), ); await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing referenced project to environment"); await stageConfinedSyncIn({ files: [{ sourcePath: localPath, targetPath: remoteProjectDir, kind: "directory", exclude, access: "ro", }], sourceRoots: [localPath], targetRoots: [remoteProjectDir], progressLabel: label, statusPhase: "config_sync", progressBytes: 0, }); additionalSourceDirs[projectId] = remoteProjectDir; } catch (error) { const message = error instanceof Error ? error.message : String(error); // Record the failure as a first-class per-project outcome so the run can count it in the // requested-vs-synced accounting and surface the reason on the run result and the run log. // The structured slot carries the reason, so no `console.warn` line is needed here. additionalFailureSlots[sourceIndex] = { projectId, error: message }; } }), ); }); // Run every inbound operation through the shared scheduler. It starts the // tasks under the client's concurrency permission and the bound, and it // settles every started task before it returns. This is the startup barrier: // the coordinator returns only after every upload settles. const inboundResults = await runStepSpan("workspace.inbound_barrier", () => scheduleSyncOperations( inboundTasks, input.client.allowConcurrentSyncOperations === true, SYNC_OPERATION_CONCURRENCY_LIMIT, )); // Collect the referenced-project failures in stable input order. for (const failure of additionalFailureSlots) { if (failure) { additionalSourceFailures.push(failure); } } // Select a fatal failure in stable operation order. The workspace comes first, // then each asset. A referenced-project task never rejects, so a nonfatal // outcome never appears here. Raise the first required rejection, so two // required failures raise the earlier one. for (let index = 0; index < inboundResults.length; index += 1) { const result = inboundResults[index]; if (inboundTaskIsRequired[index] && result.status === "rejected") { throw result.reason; } } }); const assetDirs = Object.fromEntries( (input.assets ?? []).map((asset) => [asset.key, path.posix.join(runtimeRootDir, asset.key)]), ); return { spec: input.spec, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir, runtimeRootDir, assetDirs, additionalSourceDirs, additionalSourceFailures, workspaceSyncSnapshot: syncWorkspace && baselineSnapshot ? { baseline: baselineSnapshot, gitSnapshot, ...(!gitSnapshot && repositories.length ? { repositories } : {}) } : null, cleanupWorkspaceSnapshot, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { try { const restoreSink = onProgress ?? input.onProgress; // Build the ordered outbound restore task list. Each task runs one // restore from start to end: the workspace restore, or one asset restore. // The shared scheduler starts the tasks. When the sync client permits // concurrency, the scheduler keeps at most SYNC_OPERATION_CONCURRENCY_LIMIT // tasks active. When the client does not permit concurrency, the scheduler // runs one task at a time in this order. The scheduler settles every // started task before it returns, so lease teardown never starts while an // outbound restore task still writes host data. const outboundTasks: Array> = []; const outboundDiagnostics: Array = []; // The workspace restore task runs only when the run syncs the workspace. // The task exports the sandbox git history (git-backed workspace), reads // the sandbox workspace back, and merges it into the host workspace root. // The merge is the only outbound write inside the host workspace root. // Every other task writes a disjoint host target: an asset restore writes // its own store outside the workspace root. So the workspace task and the // asset tasks share one parallel set. Keep any future asset that must write // inside the host workspace root out of this set, and run it after the // merge. Each task gets its own restore temp directory, so two concurrent // tasks never share scratch state. if (syncWorkspace) { outboundTasks.push(() => withWorkspaceRestoreDiagnostics("workspace", () => runStepSpan("restore.workspace", async () => { // Each repository owns its Git history and merge. The parent baseline also // records child files so restart recovery has their original merge inputs. for (const repository of repositories) { const prefix = `${repository.path}/`; const localDir = path.join(input.workspaceLocalDir, repository.path); if (await fs.realpath(localDir) !== path.join(await fs.realpath(input.workspaceLocalDir), repository.path)) { throw new Error("Project repository escaped its workspace"); } const nestedExclude = mergeExcludes( baselineSnapshot!.exclude.flatMap((entry) => entry.startsWith(prefix) ? [entry.slice(prefix.length)] : entry.startsWith("*/") ? [entry] : []), ); const nested = await prepareSandboxManagedRuntime({ spec: { ...input.spec, remoteCwd: path.posix.join(workspaceRemoteDir, repository.path) }, client: input.client, adapterKey: input.adapterKey, workspaceLocalDir: localDir, workspaceInboundMode: "adopt_remote", workspaceSnapshotIsolation: input.workspaceSnapshotIsolation, workspaceCheckpoint: input.workspaceCheckpoint, onWorkspaceCheckpoint: input.onWorkspaceCheckpoint, workspaceGitSnapshot: repository.snapshot, workspaceFileMode: input.workspaceFileMode, workspaceExclude: nestedExclude, workspaceBaseline: await selectDirectorySnapshot(baselineSnapshot!, { prefix, exclude: mergeExcludes(input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], nestedExclude), ignoredPaths: input.workspaceFileMode === "all" ? undefined : repository.snapshot.ignoredPaths, }), onRuntimeProgress: input.onRuntimeProgress, workspaceRestoreRetryDelaysMs: input.workspaceRestoreRetryDelaysMs, }); await nested.restoreWorkspace(restoreSink); } await retryWorkspaceRestoreBeforeHostWrite( input.workspaceRestoreRetryDelaysMs ?? WORKSPACE_RESTORE_RETRY_DELAYS_MS, (markHostWriteStarted) => withTempDir("paperclip-sandbox-restore-", async (tempDir) => { let importedRef: string | null = null; let importedHead: string | null = null; let remoteWorkspaceStatus = "dirty"; let checkpointSnapshot: DirectorySnapshot | undefined; try { if (gitSnapshot) { await emitRuntimeStatus(input.onRuntimeProgress, "export", "Exporting git changes from environment"); importedRef = createImportedGitRef("sandbox"); const remoteGitBundle = path.posix.join(runtimeRootDir, "git-delta.bundle"); const remoteWorkspaceStatusPath = path.posix.join(runtimeRootDir, "workspace-status.txt"); const exportRef = createRemoteGitExportRef("sandbox"); const localBundlePath = path.join(tempDir, "git-delta.bundle"); const remoteShallowPath = path.posix.join(runtimeRootDir, "git-shallow.txt"); const localShallowPath = path.join(tempDir, "git-shallow.txt"); // Export the sandbox history and import it into the host workspace. // The delta bundle assumes the host holds the bundle's boundary // commit; when the host has been reset far enough that it does not, // the import fails on a missing prerequisite. In that case re-export // a full, self-contained bundle from the still-live sandbox rather // than discard the completed run. const exportAndImport = async (forceFullBundle: boolean): Promise => withWorkspaceRestoreStep("git_export", async () => { try { await input.client.run( `sh -c ${shellQuote(buildRemoteGitDeltaBundleScript({ remoteDir: workspaceRemoteDir, baseSha: gitSnapshot.headCommit, exportRef, bundlePath: remoteGitBundle, statusPath: forceFullBundle ? undefined : remoteWorkspaceStatusPath, forceFullBundle, shallowPath: forceFullBundle ? remoteShallowPath : undefined, }))}`, { timeoutMs: input.spec.timeoutMs }, ); } catch (error) { // Restore the owned producer refusal across clients that // expose process status rather than the typed host error. const status = error as { exitCode?: unknown; code?: unknown; timedOut?: unknown } | null; if (status?.timedOut !== true && (status?.exitCode === GIT_SHALLOW_METADATA_EXIT_CODE || status?.code === GIT_SHALLOW_METADATA_EXIT_CODE)) throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (> ${GIT_SHALLOW_MAX_BYTES})`); throw error; } const gitExport = makeTransferProgress( restoreSink, "Exporting git history", "from", undefined, { sink: input.onRuntimeProgress, phase: "export" }, ); if (nativeSyncOut) { // Native outbound: the provider copies the bundle straight from // the sandbox into the host restore temp directory. This maps to // one `kind: "file"` mapping. The host does not buffer the full // bundle in RAM and does not move the bytes through the base64 // read loop. The git import step below reads the bundle from // `localBundlePath`. The provider transfer reports its own byte // total, so the "Exporting git history" progress still degrades to // start-and-finish (mirrors the workspace restore below), but the // finish line carries the real transferred byte count. const operations: SandboxSyncOperation[] = [{ operationId: nextSyncOperationId(), files: [{ sourcePath: remoteGitBundle, targetPath: localBundlePath, kind: "file", }], }]; assertSyncOperationsConfined(operations, { sourceRoots: [runtimeRootDir], targetRoots: [tempDir], }); const syncResult = await input.client.syncOut!(operations); const transferredBytes = sumSyncResultBytes(syncResult); await gitExport.finish(transferredBytes, transferredBytes); } else { const bundleBytes = await input.client.readFile(remoteGitBundle, gitExport.options); const bundleBuffer = toBuffer(bundleBytes); await gitExport.finish(bundleBuffer.byteLength, bundleBuffer.byteLength); await fs.writeFile(localBundlePath, bundleBuffer); } await input.client.remove(remoteGitBundle).catch(() => undefined); if (forceFullBundle) { const shallow = toBuffer(await input.client.readFile(remoteShallowPath, { onProgress: (transferred, total) => { if (transferred > GIT_SHALLOW_MAX_BYTES || (total !== null && total > GIT_SHALLOW_MAX_BYTES)) { throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${Math.max(transferred, total ?? 0)} > ${GIT_SHALLOW_MAX_BYTES})`); } }, })); if (shallow.byteLength > GIT_SHALLOW_MAX_BYTES) throw new GitShallowMetadataError(`Git shallow metadata exceeds its byte limit (${shallow.byteLength} > ${GIT_SHALLOW_MAX_BYTES})`); await fs.writeFile(localShallowPath, shallow); await input.client.remove(remoteShallowPath).catch(() => undefined); } if (!forceFullBundle) { remoteWorkspaceStatus = await input.client.readFile(remoteWorkspaceStatusPath) .then((bytes) => toBuffer(bytes).toString("utf8").trim()) .catch(() => "dirty"); remoteWorkspaceStatus = remoteWorkspaceStatus === "clean" ? "clean" : "dirty"; await input.client.remove(remoteWorkspaceStatusPath).catch(() => undefined); } return withWorkspaceRestoreStep("git_import", () => fetchGitBundleIntoLocalRef({ localDir: input.workspaceLocalDir, bundlePath: localBundlePath, exportRef, importedRef: importedRef!, baseSha: gitSnapshot.headCommit, shallowPath: forceFullBundle ? localShallowPath : undefined, })); }); try { importedHead = await exportAndImport(false); } catch (error) { if (!isMissingGitPrerequisiteError(error)) throw error; importedHead = await exportAndImport(true); } } await emitRuntimeStatus(input.onRuntimeProgress, "restore", "Restoring workspace from environment"); const extractedDir = path.join(tempDir, "workspace"); let sparseSupported = false; if (input.workspaceCheckpoint && nativeSyncOut) { sparseSupported = await input.client.run( `node --input-type=module -e ${shellQuote("import {DatabaseSync} from 'node:sqlite'; new DatabaseSync(':memory:').close()")}`, { timeoutMs: 15_000 }, ).then(() => true, () => false); } if (sparseSupported) { const checkpointBaseline = repositories.length === 0 ? baselineSnapshot! : await selectDirectorySnapshot(baselineSnapshot!, { omit: repositories.map((repo) => repo.path), exclude: workspaceRestoreExclude, ignoredPaths, }); try { const inputManifest = path.join(tempDir, "checkpoint-baseline.sqlite"); const helper = path.join(tempDir, "checkpoint.mjs"); await writeCheckpointBaseline(checkpointBaseline, inputManifest); await fs.writeFile(helper, workspaceCheckpointScript()); const remoteRoot = path.posix.join(runtimeRootDir, `checkpoint-${randomUUID()}`); const remoteManifest = path.posix.join(remoteRoot, "baseline.sqlite"); const remoteHelper = path.posix.join(remoteRoot, "checkpoint.mjs"); const remoteOutput = path.posix.join(remoteRoot, "output"); const inbound: SandboxSyncOperation[] = [{ operationId: nextSyncOperationId(), files: [ { sourcePath: inputManifest, targetPath: remoteManifest, kind: "file" }, { sourcePath: helper, targetPath: remoteHelper, kind: "file" }, ] }]; assertSyncOperationsConfined(inbound, { sourceRoots: [tempDir], targetRoots: [runtimeRootDir] }); await input.client.syncIn!(inbound); try { let captured = false; try { await input.client.run(["node", remoteHelper, workspaceRemoteDir, remoteManifest, remoteOutput, JSON.stringify(workspaceRestoreExclude)].map(shellQuote).join(" "), { timeoutMs: input.spec.timeoutMs }); captured = true; } catch (error) { if (!isCheckpointCaptureCapacityError(error)) throw error; } if (captured) { const output = path.join(tempDir, "checkpoint"); const outbound: SandboxSyncOperation[] = [{ operationId: nextSyncOperationId(), files: [ { sourcePath: remoteOutput, targetPath: output, kind: "directory" }, ] }]; assertSyncOperationsConfined(outbound, { sourceRoots: [runtimeRootDir], targetRoots: [tempDir] }); await withWorkspaceRestoreStep("workspace_transfer", () => input.client.syncOut!(outbound)); checkpointSnapshot = readCheckpointSnapshot(path.join(output, "manifest.sqlite"), checkpointBaseline); await fs.rename(path.join(output, "payload"), extractedDir); const metrics = await validateCheckpointPayload({ baseline: checkpointBaseline, snapshot: checkpointSnapshot, payload: extractedDir }); await input.onWorkspaceCheckpoint?.(metrics).catch(() => undefined); } } catch (error) { if (error instanceof Error && error.message.includes("Unsafe workspace archive symlink")) Object.assign(error, { code: "WORKSPACE_RESTORE_UNSAFE_ARCHIVE" }); throw error; } finally { await input.client.remove(remoteRoot).catch(() => undefined); } } finally { if (checkpointBaseline !== baselineSnapshot) await disposeDirectorySnapshot(checkpointBaseline); } } if (input.workspaceCheckpoint && !checkpointSnapshot) await input.onWorkspaceCheckpoint?.({ mode: "full", scannedEntries: null, changedFiles: null, payloadBytes: null }).catch(() => undefined); if (!checkpointSnapshot && nativeSyncOut) { // Native outbound: the provider materializes the sandbox workspace into // a fresh host directory. It is a clean destroy-then-replace into a // temp dir the orchestrator just created, so it maps exactly to a // generic directory file mapping; the host-side baseline merge below is // unchanged. The provider transfer reports its own byte total, so the // finish line carries the real transferred byte count. const operations: SandboxSyncOperation[] = [{ operationId: nextSyncOperationId(), files: [{ sourcePath: workspaceRemoteDir, targetPath: extractedDir, kind: "directory", exclude: workspaceRestoreExclude, }], }]; assertSyncOperationsConfined(operations, { sourceRoots: [workspaceRemoteDir], targetRoots: [extractedDir], }); await fs.mkdir(extractedDir, { recursive: true }); const workspaceRestore = makeTransferProgress( restoreSink, "Restoring", "from", "workspace", { sink: input.onRuntimeProgress, phase: "restore" }, ); const syncResult = await withWorkspaceRestoreStep("workspace_transfer", () => input.client.syncOut!(operations)); const transferredBytes = sumSyncResultBytes(syncResult); await workspaceRestore.finish(transferredBytes, transferredBytes); } else if (!checkpointSnapshot) { const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-download.tar"); await withWorkspaceRestoreStep("workspace_transfer", () => input.client.run( `sh -c ${shellQuote(createRemoteTarballFromDirectoryCommand({ remoteDir: workspaceRemoteDir, archivePath: remoteWorkspaceTar, exclude: workspaceRestoreExclude, }))}`, { timeoutMs: input.spec.timeoutMs }, )); const workspaceRestore = makeTransferProgress( restoreSink, "Restoring", "from", "workspace", { sink: input.onRuntimeProgress, phase: "restore" }, ); const archiveBytes = await withWorkspaceRestoreStep("workspace_transfer", () => input.client.readFile(remoteWorkspaceTar, workspaceRestore.options)); const archiveBuffer = toBuffer(archiveBytes); await workspaceRestore.finish(archiveBuffer.byteLength, archiveBuffer.byteLength); await input.client.remove(remoteWorkspaceTar).catch(() => undefined); const localArchivePath = path.join(tempDir, "workspace.tar"); await withWorkspaceRestoreStep("workspace_extract", async () => { await fs.writeFile(localArchivePath, archiveBuffer); await extractTarballToDirectory({ archivePath: localArchivePath, localDir: extractedDir, }); }); } if (!checkpointSnapshot) await confineFullWorkspaceExport(extractedDir); const gitHeadToIntegrate = importedHead; await withWorkspaceRestoreStep("directory_merge", async () => { const mergeBaseline = repositories.length === 0 ? baselineSnapshot! : await selectDirectorySnapshot(baselineSnapshot!, { omit: repositories.map((repo) => repo.path), exclude: workspaceRestoreExclude, ignoredPaths, }); try { await mergeDirectoryWithBaseline({ workspaceSnapshotIsolation: input.workspaceSnapshotIsolation, baseline: mergeBaseline, sourceDir: extractedDir, ...(checkpointSnapshot ? { snapshots: { source: checkpointSnapshot } } : {}), targetDir: input.workspaceLocalDir, // Runs under the merge lock, before the first host write. beforeApply: async () => { markHostWriteStarted(); if (!gitHeadToIntegrate) return; await withWorkspaceRestoreStep("git_integration", () => integrateImportedGitHead({ localDir: input.workspaceLocalDir, importedHead: gitHeadToIntegrate, baseline: gitSnapshot ?? undefined, })); }, afterApply: gitSnapshot ? async () => { await withWorkspaceRestoreStep("index_reset", () => resetLocalGitIndexToHead({ localDir: input.workspaceLocalDir, checkWorkingTreeClean: remoteWorkspaceStatus === "clean", })); } : undefined, }); } finally { if (mergeBaseline !== baselineSnapshot) await disposeDirectorySnapshot(mergeBaseline); } }); } finally { await disposeDirectorySnapshot(checkpointSnapshot ?? null); await emitRuntimeStatus(input.onRuntimeProgress, "finalize", "Finalizing workspace"); if (importedRef) { await withWorkspaceRestoreStep("git_ref_cleanup", () => deleteLocalGitRef({ localDir: input.workspaceLocalDir, ref: importedRef! })); } } })); }), restoreSink, (diagnostic) => { outboundDiagnostics[0] = diagnostic; }), ); } // One restore task per asset that has a teardown contribution. Each task // reads its asset back through `readFile` and writes a disjoint host // target. Each task gets its own restore temp directory. for (const asset of input.assets ?? []) { if (!asset.restore) continue; const assetRestore = asset.restore; const assetKey = asset.key; const taskIndex = outboundTasks.length; outboundTasks.push(() => withWorkspaceRestoreDiagnostics( "asset", () => runStepSpan(`restore.asset.${assetKey}`, async () => { await withTempDir("paperclip-sandbox-restore-", async (tempDir) => { await withWorkspaceRestoreStep("asset_restore", () => assetRestore({ assetDir: path.posix.join(runtimeRootDir, assetKey), readFile: async (remotePath) => toBuffer(await input.client.readFile(remotePath)), tempDir, })); }); }), restoreSink, (diagnostic) => { outboundDiagnostics[taskIndex] = diagnostic; }), ); } // Run every outbound restore through the shared scheduler. It starts the // tasks under the client concurrency permission and the bound, and it // settles every started task before it returns. This is the teardown // barrier. const outboundResults = await scheduleSyncOperations( outboundTasks, input.client.allowConcurrentSyncOperations === true, SYNC_OPERATION_CONCURRENCY_LIMIT, ); // Every outbound restore is a required operation: a rejection is fatal. // The workspace comes first, then each asset in order. Raise the first // rejection in stable task order, so two failures raise the earlier one. for (const [index, result] of outboundResults.entries()) { if (result.status === "rejected") { recordWorkspaceRestoreDiagnostic(result.reason, outboundDiagnostics[index]); throw result.reason; } } } finally { await cleanupWorkspaceSnapshot(); } }, }; } catch (error) { await cleanupWorkspaceSnapshot(); throw error; } }