import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { execFile as execute } from "node:child_process"; import { promisify } from "node:util"; import { tryWithDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { assertWorkspaceManifestDiskSpace } from "@paperclipai/adapter-utils/workspace-manifest"; import type { GitAuthInvocation } from "./git-credentials.js"; import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; import { recordRepositoryObjectCacheDetail, type RepositoryObjectCacheOutcome } from "./execution-preparation-timing.js"; const execFile = promisify(execute); const REPOSITORY_BYTES = 2 * 1024 ** 3; const COMPANY_BYTES = 4 * 1024 ** 3; const COMPANY_REPOSITORIES = 64; const CACHE_VERSION = 2; const hash = (value: string) => createHash("sha256").update(value).digest("hex"); class RepositoryObjectCacheSkip extends Error { constructor(readonly outcome: "capacity" | "budget" | "invalid") { super("Repository object cache skipped"); } } export interface RepositoryObjectCache { companyDirectory: string; directory: string; key: string; repoUrl: string; } export async function resolveRepositoryObjectCache(companyDirectory: string, repoUrl: string, auth: GitAuthInvocation | null): Promise { try { const remote = new URL(repoUrl); if (remote.username || remote.password || remote.search || remote.hash) return null; const parent = path.dirname(companyDirectory); await fs.mkdir(parent, { recursive: true, mode: 0o700 }); for (const directory of [parent, companyDirectory]) { if (directory === companyDirectory) await fs.mkdir(directory, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; }); const stat = await fs.lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) return null; } companyDirectory = await fs.realpath(companyDirectory); const scope = auth ? [auth.source, auth.secretName, auth.configArgs, Object.entries(auth.env).sort(([a], [b]) => a.localeCompare(b))] : null; const key = hash(JSON.stringify([CACHE_VERSION, repoUrl, scope])); return { companyDirectory, directory: path.join(companyDirectory, key), key, repoUrl }; } catch { return null; } } export async function readRepositoryObjectCache(cache: RepositoryObjectCache): Promise { try { for (const directory of [cache.companyDirectory, cache.directory]) { const stat = await fs.lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) return null; } if (await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return null; const objects = await fs.lstat(path.join(cache.directory, "objects")); if (!objects.isDirectory() || objects.isSymbolicLink()) return null; if (await fs.lstat(path.join(cache.directory, "objects/info/alternates")).then(() => true, () => false)) return null; const receiptPath = path.join(cache.directory, "paperclip-object-cache.json"); const stat = await fs.lstat(receiptPath); if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 4096) return null; const receipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); if (receipt.version !== CACHE_VERSION || receipt.key !== cache.key) return null; const options = { timeout: 10000, maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) }; const bare = await execFile("git", ["-C", cache.directory, "rev-parse", "--is-bare-repository"], options); const origin = await execFile("git", ["-C", cache.directory, "config", "--local", "--null", "--get-all", "remote.origin.url"], options); if (bare.stdout.trim() !== "true" || origin.stdout !== `${cache.repoUrl}\0`) return null; return cache.directory; } catch { return null; } } async function storageBytes(directory: string, ceiling: number, budget: { entries: number; deadline: number }): Promise { let bytes = 0; for await (const entry of await fs.opendir(directory)) { if (--budget.entries < 0 || performance.now() > budget.deadline) throw new RepositoryObjectCacheSkip("budget"); const target = path.join(directory, entry.name); const stat = await fs.lstat(target); if (stat.isSymbolicLink()) throw new RepositoryObjectCacheSkip("invalid"); bytes += stat.isDirectory() ? await storageBytes(target, ceiling - bytes, budget) : stat.size; if (bytes > ceiling) throw new RepositoryObjectCacheSkip("capacity"); } return bytes; } export async function publishRepositoryObjectCache(cache: RepositoryObjectCache, checkout: string): Promise { try { if (await readRepositoryObjectCache(cache)) { recordRepositoryObjectCacheDetail({ outcome: "already_present" }); return; } const outcome = await tryWithDirectoryMergeLock(cache.companyDirectory, async (): Promise => { const company = await fs.lstat(cache.companyDirectory); if (!company.isDirectory() || company.isSymbolicLink() || await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return "invalid"; const budget = { entries: 4096, deadline: performance.now() + 5000 }; let repositories = 0, bytes = 0; for await (const entry of await fs.opendir(cache.companyDirectory)) { if (++repositories >= COMPANY_REPOSITORIES) return "capacity"; const directory = path.join(cache.companyDirectory, entry.name); const stat = await fs.lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) return "invalid"; bytes += await storageBytes(directory, COMPANY_BYTES - bytes, budget); } if (await fs.lstat(cache.directory).then(() => true, () => false)) return "invalid"; const incoming = await storageBytes(path.join(checkout, ".git"), REPOSITORY_BYTES, budget) + 65536; if (bytes + incoming > COMPANY_BYTES) return "capacity"; assertWorkspaceManifestDiskSpace(cache.companyDirectory, incoming); const temporary = path.join(cache.companyDirectory, `.pending-${randomUUID()}`); try { const options = () => ({ timeout: Math.max(1, Math.ceil(budget.deadline - performance.now())), maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) }); await execFile("git", ["clone", "--mirror", "--no-hardlinks", "--", checkout, temporary], options()); await execFile("git", ["-C", temporary, "remote", "set-url", "origin", cache.repoUrl], options()); await fs.writeFile(path.join(temporary, "paperclip-object-cache.json"), JSON.stringify({ version: CACHE_VERSION, key: cache.key })); const actual = await storageBytes(temporary, REPOSITORY_BYTES, budget); if (bytes + actual > COMPANY_BYTES) return "capacity"; await fs.rename(temporary, cache.directory); return "published"; } finally { await fs.rm(temporary, { recursive: true, force: true }); } }); recordRepositoryObjectCacheDetail({ outcome: outcome ?? "contended" }); } catch (error) { const code = (error as NodeJS.ErrnoException)?.code; recordRepositoryObjectCacheDetail({ outcome: error instanceof RepositoryObjectCacheSkip ? error.outcome : code === "ENOSPC" || code === "workspace_git_scan_failed" ? "capacity" : "error" }); } }