diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index e5f6031d302..dc1eaa5fbde 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -594,6 +594,16 @@ If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs f Local adapters require their corresponding CLI/session setup on the machine running Paperclip. External adapters are installed through the adapter/plugin flow and should not require hardcoded imports in `server/` or `ui/`. +## Project Repository Checkouts + +Tasks use every distinct repository attached to their project, including repository-only sources with no local folder. Paperclip creates a managed checkout when no local folder is configured. The selected repository remains at the task workspace root. Other project repositories have editable, independent Git checkouts under `.paperclip-repositories/-`. Workspace hints expose each checkout path to the agent. + +When an additional repository has a configured local checkout, Paperclip seeds the task copy from its current commit and uncommitted files. Git-ignored files stay out of that copy. Subsequent task edits stay in the task copy. They do not overwrite the configured source folder. Existing task copies retain their work across runs. + +Sandbox staging, including Daytona, transfers each repository's Git history and working files. Restore merges files and commits back into each local task checkout independently. Durable sandbox recovery keeps the same repository snapshots. Normal ignore and workspace exclusion rules still apply. A clone failure stops task preparation with an error so the agent does not start with only part of the project. + +If a repository is detached or its source configuration changes, its previous task copy is retained under `.paperclip-runtime/detached-repositories/` and excluded from future sandbox transfers. Referenced projects continue to use the separate read-only multi-project workspace behavior. + ## Config Freshness Agent, project, environment, secret, skill, and workspace config edits are sampled at the next run boundary. A heartbeat that is already running finishes with the config it started with. diff --git a/doc/project-repositories.md b/doc/project-repositories.md index 779cf6104fb..38fd6e237ec 100644 --- a/doc/project-repositories.md +++ b/doc/project-repositories.md @@ -28,9 +28,13 @@ Old Overview URLs and saved Overview preferences redirect to Configuration. No schema migration is required. Selected repositories are normal project workspaces with `metadata.githubRepositoryId`. Existing manual `repoUrl` workspaces remain -editable through Configuration and the workspace API. One workspace remains primary; -additional repositories do not change the existing runtime workspace-selection or -responsible-user credential rules. A repository selection never delegates credentials. +editable through Configuration and the workspace API. One workspace remains primary. +Tasks materialize the other distinct repositories as editable checkouts inside their +workspace, including when no local folders are configured. Local execution and sandbox +staging use the same layout; sandbox restore preserves each repository's Git history. +See [Project Repository Checkouts](DEVELOPING.md#project-repository-checkouts) for paths, +ignore rules, and reuse behavior. Responsible-user credential rules still apply. +A repository selection never delegates credentials. ## Discovery and setup diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index 1b9297e7df8..3ee74500529 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -15,8 +15,12 @@ export interface GitWorkspaceSnapshot { overlayPaths: string[]; deletedPaths: string[]; ignoredPaths: string[]; + /** Managed, editable repositories inside the task workspace. */ + repositories?: Array<{ path: string; snapshot: GitWorkspaceSnapshot }>; } +export const PROJECT_REPOSITORIES_DIR = ".paperclip-repositories"; + export interface ExpensiveWorkspaceGitInput { localDir: string; args: readonly string[]; @@ -136,7 +140,25 @@ async function runExpensiveWorkspaceGit( return await runLocalGit(localDir, args, options); } -export async function readGitWorkspaceSnapshot(localDir: string): Promise { +export async function readGitWorkspaceSnapshot(localDir: string, includeRepositories = true): Promise { + const repositories: NonNullable = []; + if (includeRepositories) { + const root = path.join(localDir, PROJECT_REPOSITORIES_DIR); + const rootStat = await fs.lstat(root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (rootStat) { + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error("Invalid project repositories directory"); + for (const entry of (await fs.readdir(root, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory() || !/^[a-zA-Z0-9_-]+$/.test(entry.name)) throw new Error("Invalid project repository directory"); + const relative = `${PROJECT_REPOSITORIES_DIR}/${entry.name}`; + const snapshot = await readGitWorkspaceSnapshot(path.join(localDir, relative), false); + if (!snapshot) throw new Error(`Project repository is not a Git checkout: ${relative}`); + repositories.push({ path: relative, snapshot }); + } + } + } try { const insideWorkTree = await runLocalGit(localDir, ["rev-parse", "--is-inside-work-tree"], { timeout: 10_000, @@ -200,17 +222,22 @@ export async function readGitWorkspaceSnapshot(localDir: string): Promise repo.snapshot.overlayPaths.map((entry) => `${repo.path}/${entry}`))])] .sort((left, right) => left.localeCompare(right)), - deletedPaths: [...new Set(splitNul(deletedResult.stdout))] + deletedPaths: [...new Set([...splitNul(deletedResult.stdout), + ...repositories.flatMap((repo) => repo.snapshot.deletedPaths.map((entry) => `${repo.path}/${entry}`))])] .sort((left, right) => left.localeCompare(right)), - ignoredPaths: splitNul(ignoredResult.stdout) + ignoredPaths: [...splitNul(ignoredResult.stdout) .filter((entry) => entry.startsWith("!! ")) .map((entry) => entry.slice(3).replace(/\/+$/, "")) - .filter(Boolean) + .filter((entry) => Boolean(entry) && !(repositories.length > 0 && entry === PROJECT_REPOSITORIES_DIR)), + ...repositories.flatMap((repo) => repo.snapshot.ignoredPaths.map((entry) => `${repo.path}/${entry}`))] .sort((left, right) => left.localeCompare(right)), + ...(repositories.length > 0 ? { repositories } : {}), }; - } catch { + } catch (error) { + if (repositories.length > 0) throw error; return null; } } @@ -557,6 +584,17 @@ export async function withShallowGitWorkspaceClone( timeout: 60_000, maxBuffer: 1024 * 1024, }); + for (const repository of input.snapshot.repositories ?? []) { + await withShallowGitWorkspaceClone({ + localDir: path.join(input.localDir, repository.path), + snapshot: repository.snapshot, + }, async (nestedClone) => { + await fs.cp(nestedClone, path.join(cloneDir, repository.path), { recursive: true }); + }); + } + if (input.snapshot.repositories?.length) { + await fs.appendFile(path.join(cloneDir, ".git/info/exclude"), `\n/${PROJECT_REPOSITORIES_DIR}/\n`); + } return await fn(cloneDir); } finally { await runLocalGit(input.localDir, ["update-ref", "-d", tempRef], { diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index 387ac215dfa..d2cb0433403 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -340,6 +340,64 @@ function createRecordingTraceContext(): { describe("sandbox managed runtime", () => { const cleanupDirs: string[] = []; + it.each(["host_current", "adopt_remote", "durable_seed"] as const)("stages and restores both project repositories with independent Git histories (%s)", async (mode) => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-multi-repo-")); + cleanupDirs.push(root); + const local = path.join(root, "local"); + const remote = path.join(root, "remote"); + const secondPath = ".paperclip-repositories/backend"; + for (const [relative, contents] of [["", "frontend"], [secondPath, "backend"]]) { + const cwd = path.join(local, relative!); + await mkdir(cwd, { recursive: true }); + await git(cwd, ["init", "-b", "main"]); + await git(cwd, ["config", "user.name", "Test"]); + await git(cwd, ["config", "user.email", "test@example.com"]); + await writeFile(path.join(cwd, "README.md"), contents!); + await writeFile(path.join(cwd, ".gitignore"), "secret.txt\n"); + await git(cwd, ["add", "."]); + await git(cwd, ["commit", "-m", contents!]); + await writeFile(path.join(cwd, "secret.txt"), "must stay local"); + } + await writeFile(path.join(local, ".git/info/exclude"), ".paperclip-repositories/\n"); + await writeFile(path.join(local, secondPath, "dirty.txt"), "local edit"); + await writeFile(path.join(local, secondPath, "host-config.txt"), "excluded by operator"); + const seed = { workspaceArchivePath: path.join(root, "workspace.tar"), gitArchivePath: path.join(root, "git.tar") }; + const input = { + spec: { transport: "sandbox", provider: "test", sandboxId: "two-repos", remoteCwd: remote, timeoutMs: 30_000, apiKey: null }, + client: makeFilesystemClient(), adapterKey: "test", workspaceLocalDir: local, + workspaceDurableSeed: seed, + workspaceExclude: [`${secondPath}/host-config.txt`], + } satisfies Parameters[0]; + let prepared = await prepareSandboxManagedRuntime(input); + if (mode !== "host_current") { + if (mode === "durable_seed") await rm(remote, { recursive: true, force: true }); + await writeFile(path.join(local, secondPath, "host-only.txt"), "concurrent host work"); + prepared = await prepareSandboxManagedRuntime({ + ...input, workspaceInboundMode: mode, + workspaceBaseline: prepared.workspaceSyncSnapshot!.baseline, + workspaceGitSnapshot: prepared.workspaceSyncSnapshot!.gitSnapshot, + }); + } + for (const relative of ["", secondPath]) { + const cwd = path.join(remote, relative); + expect((await lstat(path.join(cwd, ".git"))).isDirectory()).toBe(true); + await expect(stat(path.join(cwd, "secret.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + await writeFile(path.join(cwd, "README.md"), `updated ${relative}`); + await git(cwd, ["-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-am", "remote change"]); + } + expect(await readFile(path.join(remote, secondPath, "dirty.txt"), "utf8")).toBe("local edit"); + await expect(stat(path.join(remote, secondPath, "host-config.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + await writeFile(path.join(remote, secondPath, "host-config.txt"), "remote must not replace host config"); + await prepared.restoreWorkspace(); + expect(await readFile(path.join(local, secondPath, "host-config.txt"), "utf8")).toBe("excluded by operator"); + if (mode !== "host_current") expect(await readFile(path.join(local, secondPath, "host-only.txt"), "utf8")).toBe("concurrent host work"); + for (const relative of ["", secondPath]) { + expect(await readFile(path.join(local, relative, "README.md"), "utf8")).toBe(`updated ${relative}`); + expect(await git(path.join(local, relative), ["log", "-1", "--format=%s"])).toBe("remote change"); + expect(await readFile(path.join(local, relative, "secret.txt"), "utf8")).toBe("must stay local"); + } + }, 30_000); + afterEach(async () => { while (cleanupDirs.length > 0) { const dir = cleanupDirs.pop(); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 10eb454f8df..e065e054633 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -1166,6 +1166,8 @@ export async function prepareSandboxManagedRuntime(input: { input.workspaceExclude, gitIgnoredExcludes, ); + const repositories = gitSnapshot?.repositories ?? []; + const workspaceRestoreExclude = mergeExcludes(restoreExclude, repositories.map((repo) => repo.path)); // The baseline "before" snapshot: a recursive walk of the whole workspace that // `lstat`s every entry and SHA-256-hashes every file's bytes. This is the // dominant cost in the pre-`pack` window — it reads the content of every @@ -1673,6 +1675,43 @@ export async function prepareSandboxManagedRuntime(input: { if (syncWorkspace) { outboundTasks.push(() => runStepSpan("restore.workspace", async () => { + // Each repository owns its Git history and merge. The parent baseline also + // records child files so restart recovery has their original merge inputs. + for (const repository of repositories) { + const prefix = `${repository.path}/`; + const localDir = path.join(input.workspaceLocalDir, repository.path); + if (await fs.realpath(localDir) !== path.join(await fs.realpath(input.workspaceLocalDir), repository.path)) { + throw new Error("Project repository escaped its workspace"); + } + const nestedExclude = mergeExcludes( + repository.snapshot.ignoredPaths, + baselineSnapshot!.exclude.flatMap((entry) => + entry.startsWith(prefix) ? [entry.slice(prefix.length)] + : entry.startsWith("*/") ? [entry] : []), + ); + const nested = await prepareSandboxManagedRuntime({ + spec: { ...input.spec, remoteCwd: path.posix.join(workspaceRemoteDir, repository.path) }, + client: input.client, + adapterKey: input.adapterKey, + workspaceLocalDir: localDir, + workspaceInboundMode: "adopt_remote", + workspaceGitSnapshot: repository.snapshot, + workspaceExclude: nestedExclude, + workspaceBaseline: { + exclude: mergeExcludes( + SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, + [...GIT_ARCHIVE_EXCLUDES], + [".paperclip-runtime"], + nestedExclude, + ), + entries: new Map([...baselineSnapshot!.entries] + .filter(([entry]) => entry.startsWith(prefix)) + .map(([entry, value]) => [entry.slice(prefix.length), value])), + }, + onRuntimeProgress: input.onRuntimeProgress, + }); + await nested.restoreWorkspace(restoreSink); + } await withTempDir("paperclip-sandbox-restore-", async (tempDir) => { let importedRef: string | null = null; let importedHead: string | null = null; @@ -1782,7 +1821,7 @@ export async function prepareSandboxManagedRuntime(input: { sourcePath: workspaceRemoteDir, targetPath: extractedDir, kind: "directory", - exclude: restoreExclude, + exclude: workspaceRestoreExclude, }], }]; assertSyncOperationsConfined(operations, { @@ -1806,7 +1845,7 @@ export async function prepareSandboxManagedRuntime(input: { `sh -c ${shellQuote(createRemoteTarballFromDirectoryCommand({ remoteDir: workspaceRemoteDir, archivePath: remoteWorkspaceTar, - exclude: restoreExclude, + exclude: workspaceRestoreExclude, }))}`, { timeoutMs: input.spec.timeoutMs }, ); @@ -1830,7 +1869,11 @@ export async function prepareSandboxManagedRuntime(input: { } const gitHeadToIntegrate = importedHead; await mergeDirectoryWithBaseline({ - baseline: baselineSnapshot!, + baseline: repositories.length === 0 ? baselineSnapshot! : { + exclude: workspaceRestoreExclude, + entries: new Map([...baselineSnapshot!.entries].filter(([entry]) => + !repositories.some((repo) => entry === repo.path || entry.startsWith(`${repo.path}/`)))), + }, sourceDir: extractedDir, targetDir: input.workspaceLocalDir, beforeApply: gitHeadToIntegrate diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index c2b6e44887a..fc3962b1e09 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -3406,6 +3406,13 @@ describe("applyPaperclipWorkspaceEnv", () => { }); describe("shapePaperclipWorkspaceEnvForExecution", () => { + it("maps editable project repositories inside the remote workspace", () => { + const result = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: "/host/task", executionCwd: "/sandbox/task", executionTargetIsRemote: true, + workspaceHints: [{ workspaceId: "backend", cwd: "/host/task/.paperclip-repositories/backend" }], + }); + expect(result.workspaceHints).toEqual([{ workspaceId: "backend", cwd: "/sandbox/task/.paperclip-repositories/backend" }]); + }); it("rewrites workspace env paths for remote execution", () => { const shaped = shapePaperclipWorkspaceEnvForExecution({ workspaceCwd: "/tmp/workspace", diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 6e6e4da564c..08872e9d954 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -3234,6 +3234,11 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { } return nextHint; } + const relative = localWorkspaceCwd ? path.relative(localWorkspaceCwd, hintCwd).split(path.sep).join("/") : ""; + if (realizedWorkspaceCwd && /^\.paperclip-repositories\/[a-zA-Z0-9_-]+$/.test(relative)) { + nextHint.cwd = path.posix.join(realizedWorkspaceCwd, relative); + return nextHint; + } // A referenced (mentioned) project hint carries its `projectId`. When the transport staged that // project into the sandbox, repoint the hint at its staged `project-` directory so diff --git a/server/src/__tests__/heartbeat-managed-clone-credentials.test.ts b/server/src/__tests__/heartbeat-managed-clone-credentials.test.ts index 2a0f1778852..279fb75cc4f 100644 --- a/server/src/__tests__/heartbeat-managed-clone-credentials.test.ts +++ b/server/src/__tests__/heartbeat-managed-clone-credentials.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { ensureManagedProjectWorkspace } from "../services/heartbeat.ts"; +import { ensureManagedProjectWorkspace, prepareProjectRepositoryWorkspaces } from "../services/heartbeat.ts"; import { buildGitAuthInvocation, GIT_CREDENTIAL_TOKEN_ENV_KEY } from "../services/git-credentials.ts"; import { sanitizeRuntimeServiceBaseEnv } from "../services/workspace-runtime.ts"; import { resolveManagedProjectWorkspaceDir } from "../home-paths.ts"; @@ -38,6 +38,108 @@ async function createLocalSourceRepo() { } describe("ensureManagedProjectWorkspace clone credentials", () => { + it("materializes every repository-only project row inside the task workspace and reuses local edits", async () => { + const first = await createLocalSourceRepo(); + const second = await createLocalSourceRepo(); + try { + const anchor = await ensureManagedProjectWorkspace({ companyId: "repo-only", projectId: "two", repoUrl: first }); + await execFile("git", ["checkout", "-b", "project-branch"], { cwd: second }); + const resolveGitAuth = vi.fn(async () => null); + const input = { + cwd: anchor.cwd, anchorRepoUrl: first, resolveGitAuth, + workspaces: [ + { id: "first", repoUrl: first, repoRef: null }, + { id: "duplicate-first", repoUrl: first, repoRef: null }, + { id: "second", repoUrl: second, repoRef: "project-branch" }, + ], + }; + const additional = await prepareProjectRepositoryWorkspaces(input); + expect(additional).toHaveLength(1); + expect(additional[0]!.workspaceId).toBe("second"); + expect(path.relative(anchor.cwd, additional[0]!.cwd)).toMatch(/^\.paperclip-repositories\//); + expect((await execFile("git", ["branch", "--show-current"], { cwd: additional[0]!.cwd })).stdout.trim()).toBe("project-branch"); + expect(resolveGitAuth).toHaveBeenCalledWith(second); + await fs.writeFile(path.join(additional[0]!.cwd, "README.md"), "work in progress"); + expect(await prepareProjectRepositoryWorkspaces(input)).toEqual(additional); + expect(await fs.readFile(path.join(additional[0]!.cwd, "README.md"), "utf8")).toBe("work in progress"); + expect((await execFile("git", ["status", "--porcelain"], { cwd: anchor.cwd })).stdout).toBe(""); + } finally { + await Promise.all([first, second].map((cwd) => fs.rm(cwd, { recursive: true, force: true }))); + } + }); + + it("fails task preparation when any attached repository cannot be cloned", async () => { + const first = await createLocalSourceRepo(); + try { + const anchor = await ensureManagedProjectWorkspace({ companyId: "repo-failure", projectId: "two", repoUrl: first }); + await expect(prepareProjectRepositoryWorkspaces({ + cwd: anchor.cwd, anchorRepoUrl: first, + workspaces: [{ id: "missing", repoUrl: path.join(first, "missing.git"), repoRef: null }], + })).rejects.toThrow("Failed to prepare managed checkout"); + } finally { + await fs.rm(first, { recursive: true, force: true }); + } + }); + + it("seeds a configured second local checkout with its uncommitted work and ignores", async () => { + const first = await createLocalSourceRepo(); + const second = await createLocalSourceRepo(); + try { + const anchor = await ensureManagedProjectWorkspace({ companyId: "local-project", projectId: "two", repoUrl: first }); + await fs.writeFile(path.join(second, "README.md"), "local changes"); + await fs.writeFile(path.join(second, ".gitignore"), "secret.txt\n"); + await fs.writeFile(path.join(second, "secret.txt"), "private"); + const [repo] = await prepareProjectRepositoryWorkspaces({ + cwd: anchor.cwd, anchorRepoUrl: first, + workspaces: [{ id: "second", cwd: second, repoUrl: "https://github.com/example/backend.git", repoRef: null }], + }); + expect(await fs.readFile(path.join(repo!.cwd, "README.md"), "utf8")).toBe("local changes"); + await expect(fs.stat(path.join(repo!.cwd, "secret.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + expect((await execFile("git", ["remote", "get-url", "origin"], { cwd: repo!.cwd })).stdout.trim()).toBe("https://github.com/example/backend.git"); + } finally { + await Promise.all([first, second].map((cwd) => fs.rm(cwd, { recursive: true, force: true }))); + } + }); + it("keeps different repositories with the same name separate within one project", async () => { + const first = await createLocalSourceRepo(); + const second = await createLocalSourceRepo(); + try { + const results = await Promise.all([first, second].map((repoUrl) => ensureManagedProjectWorkspace({ + companyId: "company-multiple", projectId: "project-multiple", repoUrl, + }))); + expect(results[0]!.cwd).not.toBe(results[1]!.cwd); + for (let i = 0; i < results.length; i++) { + const origin = await execFile("git", ["remote", "get-url", "origin"], { cwd: results[i]!.cwd }); + expect(origin.stdout.trim()).toBe([first, second][i]); + } + } finally { + await Promise.all([first, second].map((cwd) => fs.rm(cwd, { recursive: true, force: true }))); + } + }); + it("rechecks the repository when another process wins the checkout rename", async () => { + const first = await createLocalSourceRepo(); + const second = await createLocalSourceRepo(); + const companyId = "cross-process-race"; + const projectId = "same-name"; + const sharedCwd = resolveManagedProjectWorkspaceDir({ companyId, projectId }); + try { + // A different process does not share managedCheckoutMaterializations. Publish + // its completed checkout after this caller chose its destination, before rename. + const resolveGitAuth = vi.fn(async () => { + if (!(await fs.stat(sharedCwd).catch(() => null))) { + await execFile("git", ["clone", second, sharedCwd]); + } + return null; + }); + const result = await ensureManagedProjectWorkspace({ companyId, projectId, repoUrl: first, resolveGitAuth }); + expect((await execFile("git", ["remote", "get-url", "origin"], { cwd: result.cwd })).stdout.trim()).toBe(first); + expect((await execFile("git", ["remote", "get-url", "origin"], { cwd: sharedCwd })).stdout.trim()).toBe(second); + expect(result.cwd).not.toBe(sharedCwd); + } finally { + await Promise.all([first, second].map((cwd) => fs.rm(cwd, { recursive: true, force: true }))); + } + }); + it("clones exactly as before when no auth provider is configured", async () => { const sourceRepo = await createLocalSourceRepo(); try { diff --git a/server/src/__tests__/heartbeat-project-repositories.test.ts b/server/src/__tests__/heartbeat-project-repositories.test.ts new file mode 100644 index 00000000000..cd618f0116c --- /dev/null +++ b/server/src/__tests__/heartbeat-project-repositories.test.ts @@ -0,0 +1,84 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, writeFile, readFile, realpath, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, issues, projects, projectWorkspaces } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; + +const execute = vi.hoisted(() => vi.fn(async (_input: any) => ({ exitCode: 0, signal: null, timedOut: false }))); +vi.mock("../adapters/index.js", () => ({ + getServerAdapter: () => ({ type: "codex_local", execute, supportsLocalAgentJwt: false }), + findActiveServerAdapter: () => ({ type: "codex_local", execute, supportsLocalAgentJwt: false }), + runningProcesses: new Map(), +})); + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +suite("task project repository provisioning", () => { + let database: Awaited>; + let db: ReturnType; + let root: string; + let heartbeat: ReturnType; + beforeAll(async () => { + root = await mkdtemp(path.join(os.tmpdir(), "paperclip-project-repos-")); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); + vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "false"); + database = await startEmbeddedPostgresTestDatabase("project-repositories"); + db = createDb(database.connectionString); + heartbeat = heartbeatService(db); + execute.mockImplementation(async (input) => { + await db.update(issues).set({ status: "done" }).where(eq(issues.id, input.context.issueId)); + return { exitCode: 0, signal: null, timedOut: false }; + }); + }, 30_000); + afterAll(async () => { + if (db && heartbeat) await drainHeartbeatRunsToQuiescence(db, heartbeat); + await db?.$client.end({ timeout: 5 }); + await database?.cleanup(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); + }, 60_000); + + it.each([1, 2])("gives a task all %i repositories without any configured local folders", async (count) => { + const companyId = randomUUID(); + const projectId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const repositoryRows = []; + for (let index = 0; index < count; index++) { + const source = path.join(root, companyId, `source-${index}`); + await mkdir(source, { recursive: true }); + const git = (...args: string[]) => execFileSync("git", args, { cwd: source, stdio: "ignore" }); + git("init", "-b", "main"); + await writeFile(path.join(source, "README.md"), `repository ${index}`); + git("add", "."); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "seed"); + repositoryRows.push({ id: randomUUID(), companyId, projectId, name: `Repo ${index}`, sourceType: "git_repo", repoUrl: pathToFileURL(source).href, cwd: null, isPrimary: index === 0 }); + } + await db.insert(companies).values({ id: companyId, name: "Repo test", issuePrefix: `R${companyId.slice(0, 6)}`, defaultResponsibleUserId: "responsible-user" }); + await db.insert(projects).values({ id: projectId, companyId, name: "Multi-repo", status: "in_progress" }); + await db.insert(projectWorkspaces).values(repositoryRows); + await db.insert(agents).values({ id: agentId, companyId, name: "Test", role: "engineer", status: "idle", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, projectId, title: "Use project repositories", status: "todo", assigneeAgentId: agentId }); + const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } }); + expect(run).not.toBeNull(); + await vi.waitFor(async () => { + const latest = await heartbeat.getRun(run!.id); + expect({ status: latest?.status, error: latest?.error }).toEqual({ status: "succeeded", error: null }); + }, { timeout: 15_000 }); + const input = execute.mock.calls.find(([ctx]) => ctx.runId === run!.id)![0]; + const hints = input.context.paperclipWorkspaces as Array<{ workspaceId: string; cwd: string }>; + for (let index = 0; index < count; index++) { + const hint = hints.find((entry) => entry.workspaceId === repositoryRows[index]!.id); + expect(hint?.cwd).toBeTruthy(); + expect(await readFile(path.join(hint!.cwd, "README.md"), "utf8")).toBe(`repository ${index}`); + expect(execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd: hint!.cwd, encoding: "utf8" }).trim()).toBe(await realpath(hint!.cwd)); + } + }, 25_000); +}); diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.ts index c4598fc7086..508e1737598 100644 --- a/server/src/__tests__/native-workspace-sync.test.ts +++ b/server/src/__tests__/native-workspace-sync.test.ts @@ -134,7 +134,7 @@ describe("native workspace sync durable metadata", () => { ).toThrow("native_workspace_sync_descriptor_digest_invalid"); }); - it("writes one immutable descriptor when the same state is replayed", async () => { + it.each([false, true])("writes one immutable descriptor when the same state is replayed (multiple repositories: %s)", async (multipleRepositories) => { const paperclipHome = await mkdtemp( path.join(os.tmpdir(), "paperclip-native-workspace-sync-"), ); @@ -164,7 +164,12 @@ describe("native workspace sync durable metadata", () => { state: "prepared" as const, baselineSha256: directorySnapshotSha256(baseline), baseline: serializeDirectorySnapshot(baseline), - gitSnapshot: null, + gitSnapshot: multipleRepositories ? { + headCommit: "a".repeat(40), branchName: "main", overlayPaths: [], deletedPaths: [], ignoredPaths: [], + repositories: [{ path: ".paperclip-repositories/backend", snapshot: { + headCommit: "b".repeat(40), branchName: "backend-work", overlayPaths: ["dirty.txt"], deletedPaths: [], ignoredPaths: ["secret.txt"], + } }], + } : null, seed: null, createdAt: "2026-01-01T00:00:00.000Z", finalizedAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index e2bb3fc2be8..6c40967b852 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -28,6 +28,8 @@ import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status- export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; +import { PROJECT_REPOSITORIES_DIR, readGitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { initializeRunIdentity, explicitOperatorRunIdentity } from "./run-identity.js"; import { assertDurableChatWakeupReceipt, @@ -2372,24 +2374,49 @@ export async function ensureManagedProjectWorkspace(input: { /** Optional git credential source for cloning private repos; null/absent preserves ambient behavior. */ resolveGitAuth?: GitRemoteAuthProvider | null; }): Promise<{ cwd: string; warning: string | null }> { - const cwd = resolveManagedProjectWorkspaceDir({ + const defaultCwd = resolveManagedProjectWorkspaceDir({ companyId: input.companyId, projectId: input.projectId, repoName: deriveRepoNameFromRepoUrl(input.repoUrl), }); + let cwd = defaultCwd; + if (input.repoUrl && await fs.stat(path.join(cwd, ".git")).catch(() => null)) { + const origin = await execFile("git", ["-C", cwd, "remote", "get-url", "origin"], { timeout: 10_000 }) + .then((result) => result.stdout.trim()).catch(() => null); + if (origin && origin !== input.repoUrl) { + cwd = `${cwd}-${createHash("sha256").update(input.repoUrl).digest("hex").slice(0, 12)}`; + } + } const inFlight = managedCheckoutMaterializations.get(cwd); - if (inFlight) return inFlight; + if (inFlight) { + await inFlight; + // The winner may have cloned a different repository with the same basename. + return ensureManagedProjectWorkspace(input); + } const attempt = materializeManagedProjectWorkspace(cwd, input).finally(() => { managedCheckoutMaterializations.delete(cwd); }); managedCheckoutMaterializations.set(cwd, attempt); - return attempt; + const result = await attempt; + if (input.repoUrl) { + // A different server process can publish a same-name checkout between the + // initial origin check and the atomic rename. Never adopt its other repo. + const origin = await execFile("git", ["-C", cwd, "remote", "get-url", "origin"], { timeout: 10_000 }) + .then((value) => value.stdout.trim()).catch(() => null); + if (origin && origin !== input.repoUrl) { + if (cwd !== defaultCwd) throw new Error("Managed checkout origin does not match the requested repository"); + return ensureManagedProjectWorkspace(input); + } + } + return result; } async function materializeManagedProjectWorkspace( cwd: string, input: { repoUrl: string | null; + repoRef?: string | null; + localSource?: string | null; resolveGitAuth?: GitRemoteAuthProvider | null; }, ): Promise<{ cwd: string; warning: string | null }> { @@ -2427,14 +2454,14 @@ async function materializeManagedProjectWorkspace( // is never created in a partial state and never removed on failure, so a concurrent // materialization (another process, or a run racing this one) can neither adopt a broken // checkout nor lose its own completed one. - const auth = input.resolveGitAuth + const auth = input.resolveGitAuth && !input.localSource ? await input.resolveGitAuth(input.repoUrl) : null; const cloneTmpDir = await fs.mkdtemp(`${cwd}.clone-`); try { await execFile( "git", - [...(auth?.configArgs ?? []), "clone", input.repoUrl, cloneTmpDir], + [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", "--", input.localSource ?? input.repoUrl, cloneTmpDir], { env: { // Spread order matters: the sanitizer strips PAPERCLIP_*, which would remove the @@ -2448,6 +2475,15 @@ async function materializeManagedProjectWorkspace( timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS, }, ); + if (input.localSource) { + const snapshot = await readGitWorkspaceSnapshot(input.localSource, false); + if (!snapshot) throw new Error("Configured repository folder is not a Git checkout"); + const baseline = await captureDirectorySnapshot(cloneTmpDir, { exclude: [".git", ".paperclip-runtime", PROJECT_REPOSITORIES_DIR, ...snapshot.ignoredPaths] }); + await mergeDirectoryWithBaseline({ baseline, sourceDir: input.localSource, targetDir: cloneTmpDir }); + await execFile("git", ["-C", cloneTmpDir, "remote", "set-url", "origin", input.repoUrl], { timeout: 10_000 }); + } else if (input.repoRef) { + await execFile("git", ["-C", cloneTmpDir, "checkout", input.repoRef], { timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS }); + } } catch (error) { await fs .rm(cloneTmpDir, { recursive: true, force: true }) @@ -2484,6 +2520,59 @@ async function materializeManagedProjectWorkspace( return { cwd, warning: null }; } +/** Keep every distinct project repository inside the task's writable/synced root. */ +export async function prepareProjectRepositoryWorkspaces(input: { + cwd: string; + anchorRepoUrl: string | null; + workspaces: Array & { cwd?: string | null }>; + resolveGitAuth?: GitRemoteAuthProvider | null; +}): Promise> { + const identity = (url: string) => url.trim().replace(/\.git\/?$/, "").replace(/\/$/, ""); + const seen = new Set(input.anchorRepoUrl ? [identity(input.anchorRepoUrl)] : []); + const selected = input.workspaces.filter((workspace) => { + if (!workspace.repoUrl || seen.has(identity(workspace.repoUrl))) return false; + seen.add(identity(workspace.repoUrl)); + return true; + }); + const root = path.join(input.cwd, PROJECT_REPOSITORIES_DIR); + if (selected.length === 0 && !(await fs.lstat(root).catch(() => null))) return []; + await fs.mkdir(root, { recursive: true }); + if (await fs.realpath(root) !== path.join(await fs.realpath(input.cwd), PROJECT_REPOSITORIES_DIR)) { + throw new Error("Project repositories directory escapes the task workspace"); + } + const excludePath = await execFile("git", ["-C", input.cwd, "rev-parse", "--git-path", "info/exclude"], { timeout: 10_000 }) + .then((result) => path.resolve(input.cwd, result.stdout.trim())); + const exclude = await fs.readFile(excludePath, "utf8").catch(() => ""); + if (!exclude.split(/\r?\n/).includes(`/${PROJECT_REPOSITORIES_DIR}/`)) { + await fs.mkdir(path.dirname(excludePath), { recursive: true }); + await fs.appendFile(excludePath, `\n/${PROJECT_REPOSITORIES_DIR}/\n`); + } + const results = []; + for (const workspace of selected) { + const repoUrl = workspace.repoUrl!; + const name = (deriveRepoNameFromRepoUrl(repoUrl) ?? "repo").replace(/[^a-zA-Z0-9_-]/g, "-"); + const key = `${name}-${createHash("sha256").update(JSON.stringify([workspace.id, identity(repoUrl), workspace.repoRef, workspace.cwd ?? null])).digest("hex").slice(0, 12)}`; + const cwd = path.join(root, key); + const existing = await fs.lstat(cwd).catch(() => null); + if (existing && (!existing.isDirectory() || existing.isSymbolicLink())) throw new Error("Invalid project repository checkout path"); + const localSource = workspace.cwd && workspace.cwd !== REPO_ONLY_CWD_SENTINEL + && await fs.stat(workspace.cwd).then((entry) => entry.isDirectory()).catch(() => false) + ? workspace.cwd : null; + const result = await materializeManagedProjectWorkspace(cwd, { repoUrl, repoRef: workspace.repoRef, localSource, resolveGitAuth: input.resolveGitAuth }); + if (result.warning) throw new Error(result.warning); + results.push({ workspaceId: workspace.id, cwd, repoUrl, repoRef: workspace.repoRef }); + } + // Retain detached checkout work outside the synchronized repository set. + const active = new Set(results.map((repo) => path.basename(repo.cwd))); + for (const entry of await fs.readdir(root)) { + if (active.has(entry) || entry.includes(".clone-")) continue; + const retained = path.join(input.cwd, ".paperclip-runtime", "detached-repositories", randomUUID()); + await fs.mkdir(path.dirname(retained), { recursive: true }); + await fs.rename(path.join(root, entry), retained); + } + return results; +} + /** * Resolve one project workspace row to a usable cwd. The anchor path and each additional * referenced project share this step: use the configured cwd when present, otherwise clone or @@ -21576,6 +21665,25 @@ export function heartbeatService( ); } await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); + const projectRepositoryPaths: string[] = []; + if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) { + const repositoryRows = await db.select().from(projectWorkspaces).where(and( + eq(projectWorkspaces.companyId, agent.companyId), + eq(projectWorkspaces.projectId, executionWorkspace.projectId), + )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)); + const repositories = await prepareProjectRepositoryWorkspaces({ + cwd: executionWorkspace.cwd, + anchorRepoUrl: executionWorkspace.repoUrl, + workspaces: repositoryRows, + resolveGitAuth: workspaceGitAuthProvider, + }); + const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd])); + projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd))); + if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd); + resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({ + ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd, + })); + } if (persistedExecutionWorkspace) { context.executionWorkspaceId = persistedExecutionWorkspace.id; await db @@ -22913,13 +23021,17 @@ export function heartbeatService( companyId: agent.companyId, runId: run.id, issue: issueRef, - taskPrompt: + taskPrompt: [ readNonEmptyString( selectPaperclipTaskMarkdown(context, { resumedSession, }), ) ?? `# ${issueRef.identifier ?? issueRef.id}: ${issueRef.title}`, + projectRepositoryPaths.length > 0 + ? `## Project repositories\nThe task workspace also contains these editable Git repositories:\n${projectRepositoryPaths.map((repo) => `- ${repo}`).join("\n")}` + : null, + ].filter(Boolean).join("\n\n"), wakePayload: context.paperclipWake, resumedSession, conversationMode: context.conversationMode === true, diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index 2933cf9212f..c77125217bd 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -322,6 +322,7 @@ export function readNativeWorkspaceSyncReference( function parseGitSnapshot( value: unknown, + nested = false, ): GitWorkspaceSnapshot | null | undefined { if (value === null) return null; const candidate = parseObject(value); @@ -348,12 +349,26 @@ function parseGitSnapshot( ) { return undefined; } + const repositories: NonNullable = []; + if (candidate.repositories !== undefined) { + if (nested || !Array.isArray(candidate.repositories)) return undefined; + const seen = new Set(); + for (const raw of candidate.repositories) { + const repo = parseObject(raw); + if (typeof repo.path !== "string" || !/^\.paperclip-repositories\/[a-zA-Z0-9_-]+$/.test(repo.path) || seen.has(repo.path)) return undefined; + const snapshot = parseGitSnapshot(repo.snapshot, true); + if (!snapshot) return undefined; + seen.add(repo.path); + repositories.push({ path: repo.path, snapshot }); + } + } return { headCommit: candidate.headCommit, branchName: candidate.branchName as string | null, overlayPaths: [...(candidate.overlayPaths as string[])], deletedPaths: [...(candidate.deletedPaths as string[])], ignoredPaths: [...(candidate.ignoredPaths as string[])], + ...(repositories.length ? { repositories } : {}), }; }