diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index 10f0ec97169..084eb110dd9 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -24,6 +24,7 @@ export interface GitWorkspaceSnapshot { } export const PROJECT_REPOSITORIES_DIR = ".paperclip-repositories"; +export const PROJECT_REPOSITORY_STAGING_PATTERN = /^[a-zA-Z0-9_-]+\.clone-[a-zA-Z0-9_-]+$/; export interface ExpensiveWorkspaceGitInput { localDir: string; @@ -175,6 +176,15 @@ export function workspaceSnapshotTimeoutMs(): number { return Number.isFinite(configured) && configured >= 1000 ? Math.min(configured, 86_400_000) : 30 * 60_000; } +export async function listProjectRepositoryStagingPaths(localDir: string): Promise { + const entries = await fs.readdir(path.join(localDir, PROJECT_REPOSITORIES_DIR), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT" || error.code === "ENOTDIR") return []; + throw error; + }); + return entries.filter((entry) => entry.isDirectory() && PROJECT_REPOSITORY_STAGING_PATTERN.test(entry.name)) + .map((entry) => `${PROJECT_REPOSITORIES_DIR}/${entry.name}`).sort(); +} + export async function readManagedWorkspaceRepositories(localDir: string, options: { signal?: AbortSignal } = {}): Promise> { const repositories: NonNullable = []; try { @@ -187,6 +197,7 @@ export async function readManagedWorkspaceRepositories(localDir: string, options if (rootStat) { if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error("Invalid project repositories directory"); for await (const entry of await fs.opendir(root)) { + if (entry.isDirectory() && PROJECT_REPOSITORY_STAGING_PATTERN.test(entry.name)) continue; if (!entry.isDirectory() || !/^[a-zA-Z0-9_-]+$/.test(entry.name)) throw new Error("Invalid project repository directory"); const relative = `${PROJECT_REPOSITORIES_DIR}/${entry.name}`; const snapshot = await readGitWorkspaceSnapshot(path.join(localDir, relative), false, options); diff --git a/packages/adapter-utils/src/managed-clone-staging.test.ts b/packages/adapter-utils/src/managed-clone-staging.test.ts new file mode 100644 index 00000000000..90e142b236b --- /dev/null +++ b/packages/adapter-utils/src/managed-clone-staging.test.ts @@ -0,0 +1,197 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; +import { + disposeGitWorkspaceSnapshot, + listProjectRepositoryStagingPaths, + readGitWorkspaceSnapshot, + type GitWorkspaceSnapshot, +} from "./git-workspace-sync.js"; +import { + prepareSandboxManagedRuntime, + type SandboxManagedRuntimeClient, + type SandboxSyncOperation, + type SandboxSyncResult, +} from "./sandbox-managed-runtime.js"; +import { prepareRemoteManagedRuntime } from "./remote-managed-runtime.js"; +import { + buildSshEnvLabFixtureConfig, + getSshEnvLabSupport, + listLocalProjectRepositories, + runSshCommand, + startSshEnvLabFixture, + stopSshEnvLabFixture, +} from "./ssh.js"; +import { workspacePaths } from "./workspace-manifest.js"; + +const execFile = promisify(execFileCallback); +const STAGING_PATH = ".paperclip-repositories/backend.clone-Ab12Cd"; + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await execFile("git", ["-C", cwd, ...args], { maxBuffer: 32 * 1024 * 1024 }); + return stdout.trim(); +} + +async function createRepo(cwd: string, contents: string): Promise { + await mkdir(cwd, { recursive: true }); + await git(cwd, ["init", "-b", "main"]); + await git(cwd, ["config", "user.name", "Paperclip Test"]); + await git(cwd, ["config", "user.email", "test@paperclip.dev"]); + await writeFile(path.join(cwd, "README.md"), contents); + await git(cwd, ["add", "."]); + await git(cwd, ["commit", "-m", contents]); +} + +function makeFilesystemClient(): SandboxManagedRuntimeClient { + const client: SandboxManagedRuntimeClient = { + makeDir: async (remotePath) => { + await mkdir(remotePath, { recursive: true }); + }, + writeFile: async (remotePath, bytes) => { + await mkdir(path.dirname(remotePath), { recursive: true }); + await writeFile(remotePath, Buffer.from(bytes)); + }, + readFile: async (remotePath) => await readFile(remotePath), + listFiles: async (remotePath) => { + const entries = await readdir(remotePath, { withFileTypes: true }).catch(() => []); + return entries.filter((entry) => entry.isFile()).map((entry) => entry.name).sort((left, right) => left.localeCompare(right)); + }, + remove: async (remotePath) => { + await rm(remotePath, { recursive: true, force: true }); + }, + run: async (command) => { + await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 }); + }, + }; + client.syncIn = async (operations: SandboxSyncOperation[]): Promise => { + const resultOperations: SandboxSyncResult["operations"] = []; + for (const operation of operations) { + let filesTransferred = 0; + let bytesTransferred = 0; + for (const mapping of operation.files) { + const bytes = await readFile(mapping.sourcePath); + await mkdir(path.posix.dirname(mapping.targetPath), { recursive: true }); + await writeFile(mapping.targetPath, bytes); + filesTransferred += 1; + bytesTransferred += bytes.byteLength; + } + for (const command of operation.postUploadCommands ?? []) { + await execFile("sh", ["-c", command.command], { maxBuffer: 32 * 1024 * 1024 }); + } + resultOperations.push({ operationId: operation.operationId, filesTransferred, bytesTransferred }); + } + return { operations: resultOperations }; + }; + return client; +} + +describe("interrupted managed clone staging", () => { + const cleanupDirs: string[] = []; + const snapshots: GitWorkspaceSnapshot[] = []; + + afterEach(async () => { + for (const snapshot of snapshots.splice(0)) await disposeGitWorkspaceSnapshot(snapshot); + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (dir) await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + async function createWorkspace(prefix: string): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), prefix)); + cleanupDirs.push(root); + const local = path.join(root, "local"); + await createRepo(local, "frontend"); + await createRepo(path.join(local, ".paperclip-repositories/backend"), "backend"); + await writeFile(path.join(local, ".git/info/exclude"), ".paperclip-repositories/\n"); + await mkdir(path.join(local, STAGING_PATH, ".git"), { recursive: true }); + await writeFile(path.join(local, STAGING_PATH, "partial.txt"), "interrupted clone"); + return local; + } + + it("skips the staging directory when reading nested repositories", async () => { + const local = await createWorkspace("paperclip-clone-staging-snapshot-"); + + const snapshot = await readGitWorkspaceSnapshot(local); + if (snapshot) snapshots.push(snapshot); + + expect(snapshot?.repositories?.map((entry) => entry.path)).toEqual([".paperclip-repositories/backend"]); + const paths = [ + ...workspacePaths(snapshot!.overlayPaths), + ...workspacePaths(snapshot!.deletedPaths), + ...workspacePaths(snapshot!.ignoredPaths), + ]; + expect(paths.some((entry) => entry.includes(".clone-"))).toBe(false); + expect(await listProjectRepositoryStagingPaths(local)).toEqual([STAGING_PATH]); + expect(await listLocalProjectRepositories(local)).toEqual([".paperclip-repositories/backend"]); + }); + + it("still rejects an unexpected entry in the nested repositories directory", async () => { + const local = await createWorkspace("paperclip-clone-staging-invalid-"); + await mkdir(path.join(local, ".paperclip-repositories/backend.backup")); + + await expect(readGitWorkspaceSnapshot(local)).rejects.toThrow("Invalid project repository directory"); + await expect(listLocalProjectRepositories(local)).rejects.toThrow("Invalid project repository directory"); + }); + + it("neither ships nor deletes the staging directory during a sandbox round trip", async () => { + const local = await createWorkspace("paperclip-clone-staging-sandbox-"); + const remote = path.join(path.dirname(local), "remote"); + const backend = ".paperclip-repositories/backend"; + + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "clone-staging", remoteCwd: remote, timeoutMs: 30_000, apiKey: null }, + client: makeFilesystemClient(), + adapterKey: "test", + workspaceLocalDir: local, + }); + + expect(await readFile(path.join(remote, backend, "README.md"), "utf8")).toBe("backend"); + await expect(stat(path.join(remote, STAGING_PATH))).rejects.toMatchObject({ code: "ENOENT" }); + await writeFile(path.join(remote, backend, "README.md"), "updated backend"); + await prepared.restoreWorkspace(); + expect(await readFile(path.join(local, backend, "README.md"), "utf8")).toBe("updated backend"); + expect(await readFile(path.join(local, STAGING_PATH, "partial.txt"), "utf8")).toBe("interrupted clone"); + }, 30_000); + + it("neither ships nor deletes the staging directory during an SSH round trip without completed repositories", async () => { + const support = await getSshEnvLabSupport(); + if (!support.supported) return; + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-clone-staging-ssh-")); + cleanupDirs.push(root); + const local = path.join(root, "local"); + await createRepo(local, "frontend"); + await writeFile(path.join(local, ".git/info/exclude"), ".paperclip-repositories/\n"); + await mkdir(path.join(local, STAGING_PATH, ".git"), { recursive: true }); + await writeFile(path.join(local, STAGING_PATH, ".git", "HEAD"), "ref: refs/heads/main\n"); + await writeFile(path.join(local, STAGING_PATH, "partial.txt"), "interrupted clone"); + + const fixtureRoot = path.join(root, "fixture"); + await mkdir(fixtureRoot, { recursive: true }); + const fixture = await startSshEnvLabFixture({ statePath: path.join(fixtureRoot, "state.json") }); + try { + const config = await buildSshEnvLabFixtureConfig(fixture); + const prepared = await prepareRemoteManagedRuntime({ + spec: { ...config, remoteCwd: fixture.workspaceDir }, + runId: "clone-staging", + adapterKey: "test", + workspaceLocalDir: local, + }); + const remoteStaging = path.posix.join(prepared.workspaceRemoteDir, STAGING_PATH); + const listing = await runSshCommand(config, `if [ -e ${JSON.stringify(remoteStaging)} ]; then echo present; else echo absent; fi`); + expect(listing.stdout.trim()).toBe("absent"); + await runSshCommand(config, `printf "updated frontend" > ${JSON.stringify(path.posix.join(prepared.workspaceRemoteDir, "README.md"))}`); + + await prepared.restoreWorkspace(); + + expect(await readFile(path.join(local, "README.md"), "utf8")).toBe("updated frontend"); + expect(await readFile(path.join(local, STAGING_PATH, "partial.txt"), "utf8")).toBe("interrupted clone"); + expect(await readFile(path.join(local, STAGING_PATH, ".git", "HEAD"), "utf8")).toBe("ref: refs/heads/main\n"); + } finally { + await stopSshEnvLabFixture(fixture); + } + }, 60_000); +}); diff --git a/packages/adapter-utils/src/remote-managed-runtime.ts b/packages/adapter-utils/src/remote-managed-runtime.ts index 8c5d2581e18..6d331560d3d 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.ts @@ -152,9 +152,9 @@ export async function prepareRemoteManagedRuntime(input: { ? [ ...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime", - ...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : []), + ...(projectRepositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : preparedWorkspace.excludedPaths ?? []), ] - : [".paperclip-runtime", ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], + : [".paperclip-runtime", ...(preparedWorkspace.excludedPaths ?? []), ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], }) : null; const repositoryBaselines: Array<{ path: string; baselineSnapshot: DirectorySnapshot }> = []; diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index d330ee6b820..9159f855cdb 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -28,6 +28,7 @@ import { integrateImportedGitHead, readGitWorkspaceSnapshot, readManagedWorkspaceRepositories, + listProjectRepositoryStagingPaths, copyCloneTree, disposeGitWorkspaceSnapshot, ReferencedSourceIgnoreScanLimitExceededError, @@ -1390,6 +1391,7 @@ export async function prepareSandboxManagedRuntime(input: { return [`${relative}/.git`, ...repositoryGitPaths(repo.snapshot.repositories ?? [], relative)]; }); const repositoryMetadataExcludes = repositoryGitPaths(repositories); + const repositoryStagingExcludes = syncWorkspace ? await listProjectRepositoryStagingPaths(input.workspaceLocalDir) : []; let managedIgnored: Awaited> | null = null; let ignoredPaths = input.workspaceFileMode === "all" ? undefined : gitSnapshot?.ignoredPaths; if (!gitSnapshot && repositories.length) { @@ -1426,6 +1428,7 @@ export async function prepareSandboxManagedRuntime(input: { const workspaceArchiveExclude = mergeExcludes( [".paperclip-runtime"], repositoryMetadataExcludes, + repositoryStagingExcludes, input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES], input.workspaceExclude, @@ -1433,6 +1436,7 @@ export async function prepareSandboxManagedRuntime(input: { ); const restoreExclude = mergeExcludes( repositoryMetadataExcludes, + repositoryStagingExcludes, input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, input.workspaceFileMode === "all" && !gitSnapshot ? [] : [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], diff --git a/packages/adapter-utils/src/ssh.ts b/packages/adapter-utils/src/ssh.ts index 47a9c0b82b3..37472e8293d 100644 --- a/packages/adapter-utils/src/ssh.ts +++ b/packages/adapter-utils/src/ssh.ts @@ -9,7 +9,9 @@ import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js"; import { createUnrelatedHistoryGraftCommit, GIT_SYNC_COMMIT_IDENTITY_ARGS, + listProjectRepositoryStagingPaths, PROJECT_REPOSITORIES_DIR, + PROJECT_REPOSITORY_STAGING_PATTERN, readSanitizedOriginRemoteUrl, } from "./git-workspace-sync.js"; import type { RunProcessResult } from "./server-utils.js"; @@ -1100,6 +1102,7 @@ export async function listLocalProjectRepositories(localDir: string): Promise { +}): Promise<{ gitBacked: boolean; repositories?: string[]; excludedPaths?: string[] }> { const remoteDir = input.remoteDir ?? input.spec.remoteCwd; const gitSnapshot = input.workspaceFileMode === "all" ? null : await readLocalGitWorkspaceSnapshot(input.localDir); + const excludedPaths = await listProjectRepositoryStagingPaths(input.localDir); + const excludedResult = excludedPaths.length > 0 ? { excludedPaths } : {}; if (gitSnapshot) { const repositories = await listLocalProjectRepositories(input.localDir); @@ -1655,7 +1660,7 @@ export async function prepareWorkspaceForSshExecution(input: { localDir: input.localDir, remoteDir, snapshot: gitSnapshot, - exclude: repositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : [], + exclude: repositories.length > 0 ? [PROJECT_REPOSITORIES_DIR] : excludedPaths, onProgress: input.onProgress, progressLabel: "workspace", }); @@ -1675,7 +1680,7 @@ export async function prepareWorkspaceForSshExecution(input: { progressLabel: relative, }); } - return { gitBacked: true, ...(repositories.length > 0 ? { repositories } : {}) }; + return { gitBacked: true, ...(repositories.length > 0 ? { repositories } : {}), ...excludedResult }; } await clearRemoteDirectory({ @@ -1687,11 +1692,11 @@ export async function prepareWorkspaceForSshExecution(input: { spec: input.spec, localDir: input.localDir, remoteDir, - exclude: [".paperclip-runtime", ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], + exclude: [".paperclip-runtime", ...excludedPaths, ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], onProgress: input.onProgress, progressLabel: "workspace", }); - return { gitBacked: false }; + return { gitBacked: false, ...excludedResult }; } export async function restoreWorkspaceFromSshExecution(input: {