diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3f960499f76..f5b7edaa274 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2007,9 +2007,16 @@ Environment overrides: - `PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS=` sets how long the terminal-workspace reaper waits after an issue tree becomes terminal before it archives the execution workspace and deletes the worktree. A person can reopen - the work inside this window. The default is `7`. A value of `0` disables the + the work inside this window. The default is `1`. A value of `0` disables the cooldown and restores immediate reaping. A negative or non-numeric value falls back to the default. +- `PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS=` sets how long the reaper + waits after an issue tree becomes terminal before it removes regenerable files + from the workspace (see "Terminal workspace compaction" below). The default is + `1`. A value of `0` compacts on the next sweep. A negative, empty, or + non-numeric value falls back to the default. +- `PAPERCLIP_WORKSPACE_COMPACTION_ENABLED=false` turns compaction off. Any other + value keeps it on. Without `PAPERCLIP_DB_BACKUP_ALERT_FILE`, health checks look for `db-backup-to-s3.failure` in the backup directory, beside the backup directory, @@ -2319,6 +2326,55 @@ fields before disabling a connection. Verify workspace ownership, active runs, Git state, and runtime-service readiness before closing a workspace. A missing URL or old workspace timestamp alone does not prove that a row is disposable. +### Terminal workspace compaction + +A finished task often keeps its worktree for a long time: the reaper archives a +worktree only when its delivery is merged and its files are clean, and then only +after the cooldown. Most of a worktree's size is installed dependencies and build +output, not work. Compaction removes those files and keeps everything else, so a +finished task does not hold gigabytes while it waits for the reaper (or forever, +when it has no merged pull request or has leftover files). + +The reaper compacts a workspace once per terminal cycle, after +`PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS`, when all of these hold: + +- the workspace is an isolated linked Git worktree (never a shared workspace or a + project's main checkout); +- every issue in its tree is done or cancelled; +- no queued or running heartbeat run works on the tree or the workspace; +- no runtime service runs from the workspace. + +Compaction removes only directories that have a regenerable name +(`node_modules`, `dist`, `build`, `coverage`, `storybook-static`, `target`, +`.cache`, `.next`, `.nuxt`, `.parcel-cache`, `.svelte-kit`, `.turbo`, `.vite`), +that the repository's own `.gitignore` files ignore (a negated pattern, a global +excludes file, or `.git/info/exclude` does not count), and that hold no tracked +file. A directory with such a name +that does not qualify (for example a tracked `build/`) is searched in turn, so +ignored dependencies inside it still go. It never enters `.git` or +`.paperclip`, and it never follows a symlink. It checks each directory's ignore +status on its own, so dependencies below an ignored parent (for example +`vendor/tool/node_modules`) are found. Its Git reads have a deadline and stop +when the sweep stops. Tracked files, uncommitted changes, +untracked files, other ignored files (such as `.env` or local data), the branch, +and its commits all stay. The workspace row stays active. + +The removal is a rename into `.paperclip-compaction-trash` beside the worktree, +made while the reaper holds the workspace lifecycle lock. The reaper checks the +conditions above before the rename and again after it, in the same transaction. +If the task reopened or a run started in between, the directories move back. +Otherwise the trash is deleted. Trash that an interrupted compaction or a failed +delete leaves behind is deleted by a later sweep once it is more than an hour +old. The sweep checks each worktree parent directory at most every 30 minutes, +whether or not anything is left to compact. + +A compacted worktree is in the same state as a freshly created one, so when the +task resumes, whatever produced those files the first time produces them again: +reuse runs the workspace provision command (or `scripts/provision-worktree.sh`), +which reinstalls dependencies when `node_modules` is missing. Repositories without +a provision command rely on the agent's own install step, as they do for a new +worktree. + ### Browser realtime connection recovery If a browser cannot construct a WebSocket, the live-update and run transcript diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 48f91825806..804e488acad 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -1355,6 +1355,8 @@ The current app also exposes V1-supporting surfaces for: derive `deliveryState` as `merged_via_pr | merged_by_ancestry | unmerged | unknown`; terminal issue trees with a merged delivery and no active checkout run become cleanup-eligible with reason `issue_terminal` and are archived through the workspace cleanup path. Reopening the source issue records activity but does not restore that workspace. + Before that, terminal isolated worktrees are compacted: ignored, untracked dependency, build, and cache directories + are removed while the workspace, its branch, and all source changes stay, and reuse regenerates them. - task watchdog configuration and reusable watchdog issue orchestration for explicitly watched issue subtrees - routines and scheduled/API/webhook triggers - plugin installation, configuration, state, jobs, logs, webhooks, and plugin database namespace migration diff --git a/server/src/__tests__/execution-workspace-compaction.test.ts b/server/src/__tests__/execution-workspace-compaction.test.ts new file mode 100644 index 00000000000..b929a6aef0d --- /dev/null +++ b/server/src/__tests__/execution-workspace-compaction.test.ts @@ -0,0 +1,504 @@ +import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { + activityLog, + agents, + companies, + createDb, + executionWorkspaces, + heartbeatRuns, + issues, + projectWorkspaces, + projects, + workspaceRuntimeServices, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { + executionWorkspaceService, + type ExecutionWorkspaceServiceOptions, +} from "../services/execution-workspaces.ts"; +import { realizeExecutionWorkspace } from "../services/workspace-runtime.ts"; +import { WORKSPACE_COMPACTION_TRASH_DIR } from "../services/workspace-compaction.ts"; + +const execFileAsync = promisify(execFile); +const HOUR_MS = 60 * 60 * 1000; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres workspace compaction tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +async function git(cwd: string, args: string[]) { + const output = await execFileAsync("git", ["-C", cwd, ...args], { cwd }); + return output.stdout.trim(); +} + +async function exists(filePath: string) { + return fs.lstat(filePath).then(() => true, () => false); +} + +async function writeFile(filePath: string, contents: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents, "utf8"); +} + +async function createRepo() { + const repoRoot = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-compaction-"))); + await git(repoRoot, ["init"]); + await git(repoRoot, ["config", "user.name", "Paperclip Test"]); + await git(repoRoot, ["config", "user.email", "test@paperclip.local"]); + await writeFile(path.join(repoRoot, "README.md"), "# Test repo\n"); + await writeFile( + path.join(repoRoot, ".gitignore"), + ["node_modules/", "dist/", ".env", ".paperclip/", "local-data/", ""].join("\n"), + ); + await git(repoRoot, ["add", "README.md", ".gitignore"]); + await git(repoRoot, ["commit", "-m", "Initial commit"]); + await git(repoRoot, ["branch", "-M", "main"]); + return repoRoot; +} + +// Files a finished task leaves behind. The regenerable directories are the +// only paths compaction may remove; everything else is work or data. +const REGENERABLE_PATHS = ["node_modules", "packages/app/node_modules", "dist"]; +const PRESERVED_PATHS = [ + "notes/findings.md", + ".env", + "local-data/state.json", + ".paperclip/instances/default/db.txt", +]; + +async function populateWorktree(worktreePath: string) { + await writeFile(path.join(worktreePath, "node_modules/left-pad/index.js"), "module.exports = 1;\n"); + await writeFile(path.join(worktreePath, "packages/app/node_modules/dep/index.js"), "module.exports = 2;\n"); + await writeFile(path.join(worktreePath, "dist/server.js"), "built\n"); + await writeFile(path.join(worktreePath, "notes/findings.md"), "untracked agent notes\n"); + await writeFile(path.join(worktreePath, ".env"), "TOKEN=local\n"); + await writeFile(path.join(worktreePath, "local-data/state.json"), "{}\n"); + await writeFile(path.join(worktreePath, ".paperclip/instances/default/db.txt"), "instance data\n"); + await fs.appendFile(path.join(worktreePath, "README.md"), "uncommitted edit\n"); +} + +describeEmbeddedPostgres("terminal workspace compaction", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + const tempDirs = new Set(); + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-compaction-"); + db = createDb(tempDb.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterEach(async () => { + await db.delete(workspaceRuntimeServices); + await db.delete(activityLog); + await db.update(issues).set({ checkoutRunId: null, executionRunId: null, parentId: null }); + await db.delete(issues); + await db.delete(executionWorkspaces); + await db.delete(projectWorkspaces); + await db.delete(projects); + await db.delete(heartbeatRuns); + await db.delete(agents); + await db.delete(companies); + for (const dir of tempDirs) await fs.rm(dir, { recursive: true, force: true }); + tempDirs.clear(); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + function service(options: ExecutionWorkspaceServiceOptions = {}) { + return executionWorkspaceService(db, { + resolvePullRequestDetails: async () => ({ state: "unknown", headRef: null, headSha: null }), + workspaceReaperCooldownDays: 0, + workspaceCompactionDelayMs: 0, + ...options, + }); + } + + async function seedCompany() { + const companyId = randomUUID(); + const projectId = randomUUID(); + const agentId = randomUUID(); + const issuePrefix = `C${companyId.slice(0, 7).toUpperCase()}`; + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(projects).values({ id: projectId, companyId, name: "Compaction", status: "in_progress" }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Coder", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + return { companyId, projectId, agentId, issuePrefix }; + } + + async function seedWorkspace(options: { + worktreePath?: string; + repoRoot?: string; + commitOnBranch?: boolean; + populate?: boolean; + terminalAgoMs?: number; + mode?: "isolated_workspace" | "shared_workspace"; + } = {}) { + const repoRoot = options.repoRoot ?? await createRepo(); + tempDirs.add(repoRoot); + const branchName = `PAP-${Math.floor(Math.random() * 100000)}-finished-task`; + let worktreePath = options.worktreePath; + if (!worktreePath) { + worktreePath = path.join(repoRoot, ".paperclip", "worktrees", branchName); + await git(repoRoot, ["worktree", "add", "-b", branchName, worktreePath, "main"]); + } + if (options.commitOnBranch) { + await writeFile(path.join(worktreePath, "src/feature.ts"), "export const unmerged = true;\n"); + await git(worktreePath, ["add", "src/feature.ts"]); + await git(worktreePath, ["commit", "-m", "Unmerged work"]); + } + if (options.populate !== false) await populateWorktree(worktreePath); + + const company = await seedCompany(); + const projectWorkspaceId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const sourceIssueId = randomUUID(); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId: company.companyId, + projectId: company.projectId, + name: "Primary", + sourceType: "local_path", + isPrimary: true, + cwd: repoRoot, + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId: company.companyId, + projectId: company.projectId, + projectWorkspaceId, + mode: options.mode ?? "isolated_workspace", + strategyType: "git_worktree", + name: branchName, + status: "active", + cwd: worktreePath, + providerRef: worktreePath, + providerType: "git_worktree", + baseRef: "main", + branchName, + }); + await db.insert(issues).values({ + id: sourceIssueId, + companyId: company.companyId, + projectId: company.projectId, + identifier: `${company.issuePrefix}-1`, + title: "Finished task", + status: "done", + priority: "medium", + completedAt: new Date(Date.now() - (options.terminalAgoMs ?? 2 * HOUR_MS)), + executionWorkspaceId, + }); + await db + .update(executionWorkspaces) + .set({ sourceIssueId }) + .where(eq(executionWorkspaces.id, executionWorkspaceId)); + return { ...company, repoRoot, worktreePath, branchName, executionWorkspaceId, sourceIssueId }; + } + + async function workspaceStatus(executionWorkspaceId: string) { + const [row] = await db + .select({ status: executionWorkspaces.status }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, executionWorkspaceId)); + return row?.status ?? null; + } + + async function expectRegenerablePresent(worktreePath: string, present: boolean) { + for (const relativePath of REGENERABLE_PATHS) { + expect(await exists(path.join(worktreePath, relativePath)), relativePath).toBe(present); + } + } + + async function expectWorkPreserved(worktreePath: string) { + for (const relativePath of PRESERVED_PATHS) { + expect(await exists(path.join(worktreePath, relativePath)), relativePath).toBe(true); + } + expect(await fs.readFile(path.join(worktreePath, "README.md"), "utf8")).toContain("uncommitted edit"); + } + + it("compacts an undelivered no-PR workspace with leftover files and keeps every source change", async () => { + const seeded = await seedWorkspace({ commitOnBranch: true }); + const headBefore = await git(seeded.worktreePath, ["rev-parse", "HEAD"]); + const statusBefore = await git(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"]); + + const sweep = await service().sweepTerminalWorkspaces(); + + // The branch carries unmerged work, so the reaper keeps the workspace. + expect(sweep).toMatchObject({ compacted: 1, compactionFailed: 0, archived: 0, skippedUndelivered: 1 }); + expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active"); + await expectRegenerablePresent(seeded.worktreePath, false); + await expectWorkPreserved(seeded.worktreePath); + expect(await exists(path.join(seeded.worktreePath, "src/feature.ts"))).toBe(true); + expect(await git(seeded.worktreePath, ["rev-parse", "HEAD"])).toBe(headBefore); + expect(await git(seeded.repoRoot, ["rev-parse", `refs/heads/${seeded.branchName}`])).toBe(headBefore); + // Removing ignored directories never changes what Git sees as work. + expect(await git(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"])).toBe(statusBefore); + // The moved files are deleted, not parked. + const trashRoot = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR); + expect(await fs.readdir(trashRoot).catch(() => [])).toEqual([]); + + const [activity] = await db + .select({ details: activityLog.details }) + .from(activityLog) + .where(and( + eq(activityLog.entityId, seeded.executionWorkspaceId), + eq(activityLog.action, "execution_workspace.regenerable_files_removed"), + )); + expect((activity?.details as { removedPaths?: string[] } | undefined)?.removedPaths?.sort()) + .toEqual([...REGENERABLE_PATHS].sort()); + }, 30_000); + + it("compacts a delivered workspace that is still inside the archive cooldown", async () => { + const seeded = await seedWorkspace(); + await git(seeded.worktreePath, ["checkout", "--", "README.md"]); + + const sweep = await service({ workspaceReaperCooldownDays: 7 }).sweepTerminalWorkspaces(); + + expect(sweep).toMatchObject({ compacted: 1, archived: 0, skippedCooldown: 1 }); + expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active"); + await expectRegenerablePresent(seeded.worktreePath, false); + expect(await exists(path.join(seeded.worktreePath, "notes/findings.md"))).toBe(true); + }, 30_000); + + it("waits for the compaction delay after the issue tree becomes terminal", async () => { + const seeded = await seedWorkspace({ terminalAgoMs: 10 * 60 * 1000 }); + const svc = service({ workspaceReaperCooldownDays: 7, workspaceCompactionDelayMs: HOUR_MS }); + + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + // The worker wakes when the delay ends, not when the archive cooldown ends. + const next = await svc.nextTerminalWorkspaceSweepAt(); + expect(next).not.toBeNull(); + expect(next!).toBeLessThan(Date.now() + HOUR_MS); + + await db + .update(issues) + .set({ completedAt: new Date(Date.now() - 2 * HOUR_MS) }) + .where(eq(issues.id, seeded.sourceIssueId)); + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 }); + await expectRegenerablePresent(seeded.worktreePath, false); + }, 30_000); + + it("leaves files alone while a run is active on the task", async () => { + const seeded = await seedWorkspace(); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); + await db.update(issues).set({ checkoutRunId: runId }).where(eq(issues.id, seeded.sourceIssueId)); + + const sweep = await service().sweepTerminalWorkspaces(); + + expect(sweep).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); + + it("leaves files alone while a queued run targets the task through its context", async () => { + const seeded = await seedWorkspace(); + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId: seeded.companyId, + agentId: seeded.agentId, + status: "queued", + contextSnapshot: { issueId: seeded.sourceIssueId }, + }); + + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); + + it("leaves files alone while a child issue is unfinished", async () => { + const seeded = await seedWorkspace(); + await db.insert(issues).values({ + id: randomUUID(), + companyId: seeded.companyId, + projectId: seeded.projectId, + parentId: seeded.sourceIssueId, + identifier: `${seeded.issuePrefix}-2`, + title: "Follow-up", + status: "todo", + priority: "medium", + }); + + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0, skippedNonTerminalTree: 1 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); + + it("leaves files alone while a runtime service runs from the worktree", async () => { + const seeded = await seedWorkspace(); + await db.insert(workspaceRuntimeServices).values({ + id: randomUUID(), + companyId: seeded.companyId, + projectId: seeded.projectId, + executionWorkspaceId: seeded.executionWorkspaceId, + issueId: seeded.sourceIssueId, + scopeType: "execution_workspace", + serviceName: "web", + status: "running", + lifecycle: "shared", + command: "pnpm dev", + cwd: seeded.worktreePath, + provider: "local_process", + healthStatus: "healthy", + }); + + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); + + it("restores the files when the task reopens while they are being moved", async () => { + const seeded = await seedWorkspace(); + const svc = service({ + afterWorkspaceCompactionMove: async () => { + await db.update(issues).set({ status: "todo", completedAt: null }).where(eq(issues.id, seeded.sourceIssueId)); + }, + }); + + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0, compactionFailed: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + expect(await fs.readFile(path.join(seeded.worktreePath, "node_modules/left-pad/index.js"), "utf8")) + .toBe("module.exports = 1;\n"); + const trashRoot = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR); + expect(await fs.readdir(trashRoot).catch(() => [])).toEqual([]); + }, 30_000); + + it("restores the files when a run starts while they are being moved", async () => { + const seeded = await seedWorkspace(); + const svc = service({ + afterWorkspaceCompactionMove: async () => { + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId: seeded.companyId, + agentId: seeded.agentId, + status: "queued", + contextSnapshot: { issueId: seeded.sourceIssueId }, + }); + }, + }); + + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); + + it("never compacts a shared workspace or a main checkout", async () => { + const shared = await seedWorkspace({ mode: "shared_workspace" }); + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(shared.worktreePath, true); + + // A workspace row that points at the repository's main checkout (its .git + // is a directory) is never compacted, whatever its strategy says. + const repoRoot = await createRepo(); + await populateWorktree(repoRoot); + await seedWorkspace({ repoRoot, worktreePath: repoRoot, populate: false }); + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(repoRoot, true); + }, 30_000); + + it("compacts once per terminal cycle and again after the task finishes a later cycle", async () => { + const seeded = await seedWorkspace(); + const svc = service({ workspaceReaperCooldownDays: 7 }); + + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 }); + // A resumed run regenerates dependencies, and the task finishes again. + await writeFile(path.join(seeded.worktreePath, "node_modules/left-pad/index.js"), "module.exports = 1;\n"); + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + expect(await exists(path.join(seeded.worktreePath, "node_modules"))).toBe(true); + + await db + .update(issues) + .set({ completedAt: new Date(Date.now() - HOUR_MS) }) + .where(eq(issues.id, seeded.sourceIssueId)); + expect(await svc.sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 }); + expect(await exists(path.join(seeded.worktreePath, "node_modules"))).toBe(false); + }, 30_000); + + it("regenerates removed files when the task resumes in the compacted worktree", async () => { + const repoRoot = await createRepo(); + tempDirs.add(repoRoot); + const provisionCommand = "mkdir -p node_modules/left-pad && echo regenerated > node_modules/left-pad/index.js"; + const realizationInput = { + base: { + baseCwd: repoRoot, + source: "project_primary" as const, + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "main", + }, + config: { workspaceStrategy: { type: "git_worktree", baseRef: "main", provisionCommand } }, + issue: { id: "issue-1", identifier: "PAP-9", title: "Resume after compaction" }, + agent: { id: "agent-1", name: "Coder", companyId: "company-1" }, + } satisfies Parameters[0]; + const first = await realizeExecutionWorkspace(realizationInput); + expect(first.created).toBe(true); + await writeFile(path.join(first.cwd, "notes/findings.md"), "untracked agent notes\n"); + await writeFile(path.join(first.cwd, "src/feature.ts"), "export const unmerged = true;\n"); + await git(first.cwd, ["add", "src/feature.ts"]); + await git(first.cwd, ["commit", "-m", "Unmerged work"]); + const head = await git(first.cwd, ["rev-parse", "HEAD"]); + + const seeded = await seedWorkspace({ repoRoot, worktreePath: first.cwd, populate: false }); + expect(await service().sweepTerminalWorkspaces()).toMatchObject({ compacted: 1 }); + expect(await exists(path.join(first.cwd, "node_modules"))).toBe(false); + expect(await workspaceStatus(seeded.executionWorkspaceId)).toBe("active"); + + const resumed = await realizeExecutionWorkspace({ + ...realizationInput, + recordedBranchOwnership: { branchName: first.branchName!, createdByRuntime: true }, + }); + + expect(resumed.created).toBe(false); + expect(resumed.cwd).toBe(first.cwd); + expect(await fs.readFile(path.join(first.cwd, "node_modules/left-pad/index.js"), "utf8")).toBe("regenerated\n"); + expect(await fs.readFile(path.join(first.cwd, "notes/findings.md"), "utf8")).toBe("untracked agent notes\n"); + expect(await git(first.cwd, ["rev-parse", "HEAD"])).toBe(head); + }, 60_000); + + it("purges leftover trash even when the workspace has nothing left to compact", async () => { + const seeded = await seedWorkspace({ populate: false }); + const trashEntry = path.join(path.dirname(seeded.worktreePath), WORKSPACE_COMPACTION_TRASH_DIR, "interrupted"); + await writeFile(path.join(trashEntry, "0/left-pad/index.js"), "module.exports = 1;\n"); + const twoHoursAgo = new Date(Date.now() - 2 * HOUR_MS); + await fs.utimes(trashEntry, twoHoursAgo, twoHoursAgo); + + const sweep = await service().sweepTerminalWorkspaces(); + + expect(sweep).toMatchObject({ compacted: 0, compactionFailed: 0 }); + expect(await exists(trashEntry)).toBe(false); + }, 30_000); + + it("does not compact when compaction is disabled", async () => { + const seeded = await seedWorkspace(); + expect(await service({ workspaceCompactionDelayMs: null }).sweepTerminalWorkspaces()).toMatchObject({ compacted: 0 }); + await expectRegenerablePresent(seeded.worktreePath, true); + }, 30_000); +}); diff --git a/server/src/__tests__/workspace-compaction.test.ts b/server/src/__tests__/workspace-compaction.test.ts new file mode 100644 index 00000000000..eb19c828eff --- /dev/null +++ b/server/src/__tests__/workspace-compaction.test.ts @@ -0,0 +1,214 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; +import { + WORKSPACE_COMPACTION_TRASH_DIR, + deleteWorkspaceCompactionTrash, + findRegenerableWorkspaceDirs, + isLinkedGitWorktree, + moveWorkspaceDirsToTrash, + purgeStaleWorkspaceCompactionTrash, + restoreWorkspaceDirsFromTrash, +} from "../services/workspace-compaction.ts"; + +const execFileAsync = promisify(execFile); +const tempDirs: string[] = []; + +async function git(cwd: string, args: string[]) { + return (await execFileAsync("git", ["-C", cwd, ...args], { cwd })).stdout.trim(); +} + +async function write(filePath: string, contents = "x\n") { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents, "utf8"); +} + +async function createRepoWithWorktree(gitignore: string[]) { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-compaction-unit-"))); + tempDirs.push(root); + const repoRoot = path.join(root, "repo"); + await fs.mkdir(repoRoot); + await git(repoRoot, ["init", "-b", "main"]); + await git(repoRoot, ["config", "user.name", "Paperclip Test"]); + await git(repoRoot, ["config", "user.email", "test@paperclip.local"]); + await write(path.join(repoRoot, ".gitignore"), `${gitignore.join("\n")}\n`); + await git(repoRoot, ["add", ".gitignore"]); + await git(repoRoot, ["commit", "-m", "Initial commit"]); + const worktreePath = path.join(root, "worktrees", "PAP-1-task"); + await git(repoRoot, ["worktree", "add", "-b", "PAP-1-task", worktreePath, "main"]); + return { root, repoRoot, worktreePath }; +} + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) await fs.rm(dir, { recursive: true, force: true }); +}); + +describe("findRegenerableWorkspaceDirs", () => { + it("returns only ignored, untracked directories with a regenerable name", async () => { + const { worktreePath } = await createRepoWithWorktree([ + "node_modules/", "dist/", ".turbo/", ".env", "local-data/", ".paperclip/", "coverage/", + ]); + await write(path.join(worktreePath, "node_modules/a/index.js")); + await write(path.join(worktreePath, "packages/ui/node_modules/b/index.js")); + await write(path.join(worktreePath, "packages/ui/dist/ui.js")); + await write(path.join(worktreePath, ".turbo/cache")); + // Ignored, but not a regenerable name: data and secrets stay. + await write(path.join(worktreePath, ".env")); + await write(path.join(worktreePath, "local-data/state.json")); + // A regenerable name that Git does not ignore is source. + await write(path.join(worktreePath, "build/keep.txt")); + // The worktree instance data is never scanned. + await write(path.join(worktreePath, ".paperclip/node_modules/c/index.js")); + // An ignored directory with a force-added tracked file is source. + await write(path.join(worktreePath, "coverage/baseline.json")); + await git(worktreePath, ["add", "-f", "coverage/baseline.json"]); + + expect((await findRegenerableWorkspaceDirs(worktreePath)).sort()).toEqual([ + ".turbo", + "node_modules", + "packages/ui/dist", + "packages/ui/node_modules", + ]); + }); + + it("does not follow or return a symlinked directory", async () => { + const { root, worktreePath } = await createRepoWithWorktree(["node_modules"]); + const outside = path.join(root, "shared-store"); + await write(path.join(outside, "node_modules/pkg/index.js")); + await fs.symlink(path.join(outside, "node_modules"), path.join(worktreePath, "node_modules")); + await fs.symlink(outside, path.join(worktreePath, "linked")); + + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual([]); + }); + + it("searches inside a regenerable-named directory that is source", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/"]); + // `build/` holds tracked source, so it stays, but its ignored dependencies go. + await write(path.join(worktreePath, "build/scripts/release.ts")); + await git(worktreePath, ["add", "build/scripts/release.ts"]); + await write(path.join(worktreePath, "build/node_modules/tool/index.js")); + // An untracked, not-ignored `dist/` is work in progress; its ignored + // dependencies still go. + await write(path.join(worktreePath, "dist/notes.md")); + await write(path.join(worktreePath, "dist/node_modules/dep/index.js")); + + expect((await findRegenerableWorkspaceDirs(worktreePath)).sort()).toEqual([ + "build/node_modules", + "dist/node_modules", + ]); + }); + + it("finds dependencies below an ignored parent directory", async () => { + const { worktreePath } = await createRepoWithWorktree(["vendor/", "node_modules/"]); + await write(path.join(worktreePath, "vendor/tool/source.c")); + await write(path.join(worktreePath, "vendor/tool/node_modules/dep/index.js")); + // A single nested candidate once made `ls-files --directory` fail. + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["vendor/tool/node_modules"]); + + await write(path.join(worktreePath, "node_modules/a/index.js")); + expect((await findRegenerableWorkspaceDirs(worktreePath)).sort()) + .toEqual(["node_modules", "vendor/tool/node_modules"]); + }); + + it("keeps a directory that a negated pattern unignores", async () => { + const { worktreePath } = await createRepoWithWorktree(["build/", "!build/", "dist/", "node_modules/"]); + // Untracked work in an explicitly unignored directory is not regenerable. + await write(path.join(worktreePath, "build/release-notes.md")); + await write(path.join(worktreePath, "dist/bundle.js")); + + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["dist"]); + await expect(fs.access(path.join(worktreePath, "build/release-notes.md"))).resolves.toBeUndefined(); + }); + + it("handles directory names with glob characters in the tracked-file check", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/"]); + await write(path.join(worktreePath, "pkg[1]/node_modules/vendored/index.js")); + await git(worktreePath, ["add", "-f", "pkg[1]/node_modules/vendored/index.js"]); + await write(path.join(worktreePath, "pkg1/node_modules/dep/index.js")); + + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["pkg1/node_modules"]); + }); + + it("ignores global and repository-local exclude files", async () => { + const { root, repoRoot, worktreePath } = await createRepoWithWorktree(["dist/"]); + const globalExcludes = path.join(root, "global-gitignore"); + await write(globalExcludes, "build/\n"); + await git(repoRoot, ["config", "core.excludesFile", globalExcludes]); + await write(path.join(repoRoot, ".git/info/exclude"), "coverage/\n"); + await write(path.join(worktreePath, "build/draft.md")); + await write(path.join(worktreePath, "coverage/report.html")); + await write(path.join(worktreePath, "dist/bundle.js")); + + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual(["dist"]); + }); + + it("stops when the sweep is aborted", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/"]); + await write(path.join(worktreePath, "node_modules/a/index.js")); + const controller = new AbortController(); + controller.abort(); + await expect(findRegenerableWorkspaceDirs(worktreePath, { signal: controller.signal })).rejects.toThrow(); + }); + + it("returns nothing when no directory qualifies", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/"]); + await write(path.join(worktreePath, "src/index.ts")); + expect(await findRegenerableWorkspaceDirs(worktreePath)).toEqual([]); + }); +}); + +describe("isLinkedGitWorktree", () => { + it("accepts a linked worktree and rejects a main checkout", async () => { + const { repoRoot, worktreePath } = await createRepoWithWorktree([]); + expect(await isLinkedGitWorktree(worktreePath)).toBe(true); + expect(await isLinkedGitWorktree(repoRoot)).toBe(false); + }); +}); + +describe("compaction trash", () => { + it("moves directories beside the worktree and restores them unchanged", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/", "dist/"]); + await write(path.join(worktreePath, "node_modules/a/index.js"), "dependency\n"); + await write(path.join(worktreePath, "packages/ui/dist/ui.js"), "bundle\n"); + + const move = await moveWorkspaceDirsToTrash(worktreePath, ["node_modules", "packages/ui/dist", "missing"]); + expect(move.moved.map((entry) => entry.relativePath)).toEqual(["node_modules", "packages/ui/dist"]); + expect(path.dirname(path.dirname(move.trashDir))).toBe(path.dirname(worktreePath)); + expect(path.basename(path.dirname(move.trashDir))).toBe(WORKSPACE_COMPACTION_TRASH_DIR); + await expect(fs.access(path.join(worktreePath, "node_modules"))).rejects.toThrow(); + + await restoreWorkspaceDirsFromTrash(worktreePath, move); + expect(await fs.readFile(path.join(worktreePath, "node_modules/a/index.js"), "utf8")).toBe("dependency\n"); + expect(await fs.readFile(path.join(worktreePath, "packages/ui/dist/ui.js"), "utf8")).toBe("bundle\n"); + await deleteWorkspaceCompactionTrash(move.trashDir); + await expect(fs.access(move.trashDir)).rejects.toThrow(); + }); + + it("keeps a directory something recreated instead of overwriting it on restore", async () => { + const { worktreePath } = await createRepoWithWorktree(["node_modules/"]); + await write(path.join(worktreePath, "node_modules/a/index.js"), "old\n"); + const move = await moveWorkspaceDirsToTrash(worktreePath, ["node_modules"]); + await write(path.join(worktreePath, "node_modules/a/index.js"), "reinstalled\n"); + + await restoreWorkspaceDirsFromTrash(worktreePath, move); + expect(await fs.readFile(path.join(worktreePath, "node_modules/a/index.js"), "utf8")).toBe("reinstalled\n"); + }); + + it("purges only trash older than the threshold", async () => { + const { worktreePath } = await createRepoWithWorktree([]); + const parent = path.dirname(worktreePath); + const stale = path.join(parent, WORKSPACE_COMPACTION_TRASH_DIR, "stale"); + const fresh = path.join(parent, WORKSPACE_COMPACTION_TRASH_DIR, "fresh"); + await write(path.join(stale, "0/file")); + await write(path.join(fresh, "0/file")); + const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + await fs.utimes(stale, twoHoursAgo, twoHoursAgo); + + await purgeStaleWorkspaceCompactionTrash(parent, 60 * 60 * 1000); + await expect(fs.access(stale)).rejects.toThrow(); + await expect(fs.access(fresh)).resolves.toBeUndefined(); + }); +}); diff --git a/server/src/__tests__/workspace-reaper-cooldown-config.test.ts b/server/src/__tests__/workspace-reaper-cooldown-config.test.ts index 2bcde2f4d69..51d81345607 100644 --- a/server/src/__tests__/workspace-reaper-cooldown-config.test.ts +++ b/server/src/__tests__/workspace-reaper-cooldown-config.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { loadConfig } from "../config.ts"; // The terminal-workspace reaper reads PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS. -// These tests lock the parser contract: the default is 7 days, an explicit 0 +// These tests lock the parser contract: the default is 1 day, an explicit 0 // means immediate reaping, and empty, whitespace-only, negative, or non-numeric // values fall back to the default. An empty or whitespace-only value must not // become 0, because that would delete terminal workspaces immediately. @@ -12,19 +12,19 @@ describe("workspace reaper cooldown config parsing", () => { vi.unstubAllEnvs(); }); - it("uses the 7 day default when the variable is not set", () => { + it("uses the 1 day default when the variable is not set", () => { vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", undefined); - expect(loadConfig().workspaceReaperCooldownDays).toBe(7); + expect(loadConfig().workspaceReaperCooldownDays).toBe(1); }); - it("uses the 7 day default for an empty value", () => { + it("uses the 1 day default for an empty value", () => { vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", ""); - expect(loadConfig().workspaceReaperCooldownDays).toBe(7); + expect(loadConfig().workspaceReaperCooldownDays).toBe(1); }); - it("uses the 7 day default for a whitespace-only value", () => { + it("uses the 1 day default for a whitespace-only value", () => { vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", " "); - expect(loadConfig().workspaceReaperCooldownDays).toBe(7); + expect(loadConfig().workspaceReaperCooldownDays).toBe(1); }); it("keeps an explicit 0 as immediate reaping", () => { @@ -42,13 +42,50 @@ describe("workspace reaper cooldown config parsing", () => { expect(loadConfig().workspaceReaperCooldownDays).toBe(3); }); - it("uses the 7 day default for a negative value", () => { + it("uses the 1 day default for a negative value", () => { vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "-1"); - expect(loadConfig().workspaceReaperCooldownDays).toBe(7); + expect(loadConfig().workspaceReaperCooldownDays).toBe(1); }); - it("uses the 7 day default for a non-numeric value", () => { + it("uses the 1 day default for a non-numeric value", () => { vi.stubEnv("PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "soon"); - expect(loadConfig().workspaceReaperCooldownDays).toBe(7); + expect(loadConfig().workspaceReaperCooldownDays).toBe(1); + }); +}); + +// The reaper reads PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS and +// PAPERCLIP_WORKSPACE_COMPACTION_ENABLED. The default delay is 1 hour, an +// explicit 0 compacts on the next sweep, invalid values fall back to the +// default, and only an explicit "false" turns compaction off. +describe("workspace compaction config parsing", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("uses the 1 hour default when the variables are not set", () => { + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", undefined); + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", undefined); + expect(loadConfig().workspaceCompactionDelayHours).toBe(1); + }); + + it("keeps an explicit 0 and reads fractional hours", () => { + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", "0"); + expect(loadConfig().workspaceCompactionDelayHours).toBe(0); + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", " 0.5 "); + expect(loadConfig().workspaceCompactionDelayHours).toBe(0.5); + }); + + it("uses the 1 hour default for empty, negative, or non-numeric values", () => { + for (const value of ["", " ", "-2", "later"]) { + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", value); + expect(loadConfig().workspaceCompactionDelayHours).toBe(1); + } + }); + + it("turns compaction off only for an explicit false", () => { + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "false"); + expect(loadConfig().workspaceCompactionDelayHours).toBeNull(); + vi.stubEnv("PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "no"); + expect(loadConfig().workspaceCompactionDelayHours).toBe(1); }); }); diff --git a/server/src/config.ts b/server/src/config.ts index db5efb2cbda..69860d95f96 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -80,6 +80,7 @@ export interface Config { databaseBackupRetentionDays: number; databaseBackupDir: string; workspaceReaperCooldownDays: number; + workspaceCompactionDelayHours: number | null; serveUi: boolean; uiDevMiddleware: boolean; secretsProvider: SecretProvider; @@ -281,8 +282,10 @@ export function loadConfig(): Config { // becomes terminal before it archives the workspace. A person can reopen the // work inside this window. A value of 0 disables the cooldown and restores // immediate reaping. A negative or non-numeric value falls back to the - // default. The day granularity and the default of 7 obey the - // PAPERCLIP_DB_BACKUP_RETENTION_DAYS precedent above. + // default. The day granularity obeys the PAPERCLIP_DB_BACKUP_RETENTION_DAYS + // precedent above. The default is 1 day: regenerable files are compacted + // after a few hours anyway, so a longer hold only keeps source checkouts of + // delivered work, and a recurring task produces one workspace per run. const workspaceReaperCooldownDaysEnv = process.env.PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS?.trim(); const workspaceReaperCooldownDaysRaw = Number(workspaceReaperCooldownDaysEnv); @@ -291,7 +294,25 @@ export function loadConfig(): Config { && Number.isFinite(workspaceReaperCooldownDaysRaw) && workspaceReaperCooldownDaysRaw >= 0 ? workspaceReaperCooldownDaysRaw - : 7; + : 1; + // The terminal-workspace reaper removes regenerable files (installed + // dependencies, build output, tool caches) from a terminal workspace this + // many hours after its issue tree becomes terminal. The worktree and every + // source file stay; a resumed task regenerates the files when its workspace + // is provisioned again. 0 compacts on the next sweep. A negative, empty, or + // non-numeric value falls back to the default of 1 hour. + // PAPERCLIP_WORKSPACE_COMPACTION_ENABLED=false turns compaction off. + const workspaceCompactionDelayHoursEnv = + process.env.PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS?.trim(); + const workspaceCompactionDelayHoursRaw = Number(workspaceCompactionDelayHoursEnv); + const workspaceCompactionDelayHours = + process.env.PAPERCLIP_WORKSPACE_COMPACTION_ENABLED?.trim().toLowerCase() === "false" + ? null + : workspaceCompactionDelayHoursEnv + && Number.isFinite(workspaceCompactionDelayHoursRaw) + && workspaceCompactionDelayHoursRaw >= 0 + ? workspaceCompactionDelayHoursRaw + : 1; const bindValidationErrors = validateConfiguredBindMode({ deploymentMode, deploymentExposure, @@ -338,6 +359,7 @@ export function loadConfig(): Config { databaseBackupRetentionDays, databaseBackupDir, workspaceReaperCooldownDays, + workspaceCompactionDelayHours, serveUi: process.env.SERVE_UI !== undefined ? process.env.SERVE_UI === "true" diff --git a/server/src/index.ts b/server/src/index.ts index 9b5f85562b6..d2c89452af4 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1374,6 +1374,9 @@ async function startServerWithDatabaseTeardown( }); const terminalWorkspaces = executionWorkspaceService(db as any, { workspaceReaperCooldownDays: config.workspaceReaperCooldownDays, + workspaceCompactionDelayMs: config.workspaceCompactionDelayHours === null + ? null + : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); const scheduleMergedPullRequestConfirmationSweep = () => { @@ -1394,7 +1397,7 @@ async function startServerWithDatabaseTeardown( ...terminalWorkspaces, sweepTerminalWorkspaces: async (limit, signal) => { const result = await terminalWorkspaces.sweepTerminalWorkspaces(limit, signal); - if (result.archived > 0 || result.cleanupFailed > 0) { + if (result.archived > 0 || result.cleanupFailed > 0 || result.compacted > 0 || result.compactionFailed > 0) { logger.info(result, "terminal issue workspace reaper changed workspace state"); } else if (result.checked > 0 && Date.now() - lastTerminalWorkspaceSkipLogAt >= 10 * 60 * 1000) { lastTerminalWorkspaceSkipLogAt = Date.now(); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 43c990e1196..05e0bc77a63 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -12,6 +12,14 @@ import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSq import type { SQL } from "drizzle-orm"; import { createWorkspaceGitInspectionCache } from "./workspace-git-inspection-cache.js"; import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; +import { + deleteWorkspaceCompactionTrash, + findRegenerableWorkspaceDirs, + isLinkedGitWorktree, + moveWorkspaceDirsToTrash, + purgeStaleWorkspaceCompactionTrash, + restoreWorkspaceDirsFromTrash, +} from "./workspace-compaction.js"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs/promises"; @@ -282,8 +290,17 @@ export type ExecutionWorkspaceServiceOptions = { beforeTerminalWorkspaceCleanup?: (workspace: ExecutionWorkspaceRow) => Promise; // The terminal-workspace reaper waits this many days after an issue tree // becomes terminal before it archives the workspace. A value of 0 disables - // the cooldown. The default is 7 days. + // the cooldown. The default is 1 day. workspaceReaperCooldownDays?: number; + // The reaper removes regenerable files (installed dependencies, build output, + // tool caches) from a terminal workspace this long after its issue tree + // becomes terminal. The worktree, its branch, its commits, and every source + // file stay, so the task can resume in place. Unset or null disables + // compaction. + workspaceCompactionDelayMs?: number | null; + // Test hook. Runs after compaction moves directories out of the worktree and + // before it re-checks the guard under the lifecycle lock. + afterWorkspaceCompactionMove?: (workspace: ExecutionWorkspaceRow) => Promise; inspectGitCloseReadiness?: (workspace: ExecutionWorkspace) => Promise<{ git: ExecutionWorkspaceCloseGitReadiness | null; warnings: string[]; @@ -1366,6 +1383,11 @@ async function hasPendingTaskWorkspaceRecovery(tx: DbTransaction, input: { }); } +// Compaction trash older than this belongs to no compaction in progress: the +// move and the guard finish in one short transaction. +const WORKSPACE_COMPACTION_TRASH_STALE_MS = 60 * 60 * 1000; +const WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS = 30 * 60 * 1000; + export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServiceOptions = {}) { async function resolveSourceWorkspaceStrategy(input: { companyId: string; issueId?: string | null; assigneeAgentId?: string | null; @@ -1602,8 +1624,17 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // disables the cooldown. const workspaceReaperCooldownMs = Math.max( 0, - (opts.workspaceReaperCooldownDays ?? 7) * 24 * 60 * 60 * 1000, + (opts.workspaceReaperCooldownDays ?? 1) * 24 * 60 * 60 * 1000, ); + const workspaceCompactionDelayMs = typeof opts.workspaceCompactionDelayMs === "number" + ? Math.max(0, opts.workspaceCompactionDelayMs) + : null; + // The latest terminal timestamp each workspace was compacted for. A workspace + // is compacted once per terminal cycle: a resumed task can regenerate its + // files, and the next terminal cycle compacts them again. The memo is only a + // cache; after a restart compaction re-scans, finds nothing, and records it. + const compactedTerminalAt = new Map(); + const compactionTrashPurgedAt = new Map(); const pullRequestStateCache = new Map< string, { @@ -1990,6 +2021,187 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); } + // Reads, inside the caller's transaction, whether compaction may touch the + // workspace files right now: the issue tree is terminal, no queued or running + // heartbeat run works on the tree or the workspace, and no runtime service + // still runs from the worktree. + async function terminalWorkspaceCompactionGuardPasses(tx: DbTransaction, workspace: ExecutionWorkspaceRow) { + const rows = await tx.execute<{ allowed: boolean }>(sql` + WITH RECURSIVE issue_tree(id, status) AS ( + SELECT root.id, root.status + FROM ${issues} root + WHERE root.company_id = ${workspace.companyId} + AND root.id = ${workspace.sourceIssueId} + UNION ALL + SELECT child.id, child.status + FROM ${issues} child + JOIN issue_tree parent ON child.parent_id = parent.id + WHERE child.company_id = ${workspace.companyId} + ) + SELECT ( + EXISTS (SELECT 1 FROM issue_tree) + AND NOT EXISTS (SELECT 1 FROM issue_tree WHERE status NOT IN ('done', 'cancelled')) + AND NOT EXISTS ( + SELECT 1 + FROM ${issues} linked_issue + JOIN ${heartbeatRuns} live_run + ON live_run.id = linked_issue.checkout_run_id + OR live_run.id = linked_issue.execution_run_id + WHERE linked_issue.company_id = ${workspace.companyId} + AND ( + linked_issue.execution_workspace_id = ${workspace.id} + OR linked_issue.id IN (SELECT id FROM issue_tree) + ) + AND live_run.company_id = ${workspace.companyId} + AND live_run.status IN ('queued', 'running') + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${heartbeatRuns} issue_run + WHERE issue_run.company_id = ${workspace.companyId} + AND issue_run.status IN ('queued', 'running') + AND coalesce( + issue_run.native_issue_id::text, + nullif(issue_run.context_snapshot->>'issueId', ''), + nullif(issue_run.context_snapshot->>'taskId', '') + ) IN (SELECT id::text FROM issue_tree) + ) + AND NOT EXISTS ( + SELECT 1 + FROM ${workspaceRuntimeServices} service + WHERE service.company_id = ${workspace.companyId} + AND service.execution_workspace_id = ${workspace.id} + AND service.status <> 'stopped' + ) + ) AS allowed + `); + return rows[0]?.allowed === true; + } + + // Deletes compaction trash that an interrupted compaction or a failed delete + // left beside a worktree. The sweep calls this for every worktree candidate, + // independent of the per-cycle compaction memo, so leftover trash is found + // after a restart and after a workspace has nothing left to compact. Each + // worktree parent directory is checked at most once per interval. + async function purgeCompactionTrashIfDue(workspace: ExecutionWorkspaceRow) { + const worktreePath = workspace.providerRef ?? workspace.cwd; + if (workspace.providerType !== "git_worktree" || !worktreePath) return; + const parentDir = path.dirname(path.resolve(worktreePath)); + const nowMs = now().getTime(); + const lastPurgedAt = compactionTrashPurgedAt.get(parentDir); + if (lastPurgedAt !== undefined && nowMs - lastPurgedAt < WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS) return; + if (compactionTrashPurgedAt.size >= 1024) compactionTrashPurgedAt.delete(compactionTrashPurgedAt.keys().next().value!); + compactionTrashPurgedAt.set(parentDir, nowMs); + await purgeStaleWorkspaceCompactionTrash(parentDir, WORKSPACE_COMPACTION_TRASH_STALE_MS).catch((error) => { + logger.warn({ err: error, parentDir }, "failed to purge stale workspace compaction trash"); + }); + } + + // Removes regenerable files (installed dependencies, build output, tool + // caches) from a terminal workspace. The workspace row stays active and the + // worktree, its branch, its commits, and every tracked or untracked source + // file stay in place, so a resumed task continues where it stopped and its + // provisioning regenerates the removed files. This applies to workspaces the + // reaper cannot archive: an undelivered branch, leftover files, or a workspace + // still inside the archive cooldown. + async function compactTerminalWorkspace( + workspace: ExecutionWorkspaceRow, + signal?: AbortSignal, + ): Promise<"compacted" | "nothing_to_compact" | "not_compactable" | "busy"> { + const worktreePath = workspace.providerRef ?? workspace.cwd; + if ( + workspace.providerType !== "git_worktree" + || workspace.mode === "shared_workspace" + || !workspace.sourceIssueId + || !worktreePath + ) { + return "not_compactable"; + } + if (!(await isLinkedGitWorktree(worktreePath))) return "not_compactable"; + const projectWorkspaceCwd = workspace.projectWorkspaceId + ? await db + .select({ cwd: projectWorkspaces.cwd }) + .from(projectWorkspaces) + .where(and( + eq(projectWorkspaces.companyId, workspace.companyId), + eq(projectWorkspaces.id, workspace.projectWorkspaceId), + )) + .then((rows) => rows[0]?.cwd ?? null) + : null; + if (projectWorkspaceCwd && path.resolve(projectWorkspaceCwd) === path.resolve(worktreePath)) { + return "not_compactable"; + } + + const relativePaths = await findRegenerableWorkspaceDirs(worktreePath, { signal }); + if (relativePaths.length === 0) return "nothing_to_compact"; + + const snapshotGeneration = readExecutionWorkspaceLifecycleGeneration( + workspace.metadata as Record | null, + ); + // Hold the lifecycle lock while the directories move. The guard runs before + // the move and again after it, in the same transaction. A task that resumes + // in between leaves the terminal state or starts a run first, so the second + // check fails and the directories move back before anything is deleted. + const move = await db.transaction(async (tx) => { + await acquireExecutionWorkspaceLifecycleLock(tx, workspace.id); + const [fresh] = await tx + .select({ + status: executionWorkspaces.status, + closedAt: executionWorkspaces.closedAt, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where(and(eq(executionWorkspaces.id, workspace.id), eq(executionWorkspaces.companyId, workspace.companyId))); + const freshMetadata = (fresh?.metadata as Record | null) ?? null; + if ( + !fresh + || isClosedExecutionWorkspaceStatus(fresh.status) + || fresh.closedAt + || metadataHasReopenPendingConsumption(freshMetadata) + || readExecutionWorkspaceLifecycleGeneration(freshMetadata) !== snapshotGeneration + || !(await terminalWorkspaceCompactionGuardPasses(tx, workspace)) + ) { + return null; + } + const moved = await moveWorkspaceDirsToTrash(worktreePath, relativePaths); + try { + await opts.afterWorkspaceCompactionMove?.(workspace); + if (!(await terminalWorkspaceCompactionGuardPasses(tx, workspace))) { + await restoreWorkspaceDirsFromTrash(worktreePath, moved); + await deleteWorkspaceCompactionTrash(moved.trashDir); + return null; + } + } catch (error) { + await restoreWorkspaceDirsFromTrash(worktreePath, moved); + await deleteWorkspaceCompactionTrash(moved.trashDir); + throw error; + } + return moved; + }); + if (!move) return "busy"; + + await deleteWorkspaceCompactionTrash(move.trashDir).catch((error) => { + logger.warn( + { err: error, executionWorkspaceId: workspace.id, trashDir: move.trashDir }, + "failed to delete compacted workspace files; a later compaction retries", + ); + }); + if (move.moved.length === 0) return "nothing_to_compact"; + await logActivity(db, { + companyId: workspace.companyId, + actorType: "system", + actorId: "workspace_terminality_reaper", + action: "execution_workspace.regenerable_files_removed", + entityType: "execution_workspace", + entityId: workspace.id, + details: { + sourceIssueId: workspace.sourceIssueId, + removedPaths: move.moved.map((entry) => entry.relativePath), + }, + }); + return "compacted"; + } + async function cleanupTerminalWorkspace( workspace: ExecutionWorkspaceRow, expectedHeadSha: string | null, @@ -2885,6 +3097,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic skippedCooldown: 0, skippedBackoff: 0, clearedStaleReopenPending: 0, + compacted: 0, + compactionFailed: 0, }; } const previousCursor = terminalSweepCursor; @@ -2954,11 +3168,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic skippedCooldown: 0, skippedBackoff: 0, clearedStaleReopenPending: 0, + compacted: 0, + compactionFailed: 0, }; const readinessByWorkingTree = new Map>(); for (const workspace of candidates) { signal?.throwIfAborted(); + await purgeCompactionTrashIfDue(workspace); const issueTree = await listWorkspaceIssueTree(workspace); const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId); const reopenPending = metadataHasReopenPendingConsumption( @@ -2977,8 +3194,34 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic continue; } const latestTerminalAt = Math.max(...issueTree.map(issue => issueTerminalTimestamp(issue)!.getTime())); + // Compaction runs once per terminal cycle, after its own delay and + // before the archive cooldown, so a workspace the reaper cannot archive + // yet (or ever) stops holding regenerable files. + const compactionPending = workspaceCompactionDelayMs !== null + && !reopenPending + && compactedTerminalAt.get(workspace.id) !== latestTerminalAt; + const compactionDueAt = latestTerminalAt + (workspaceCompactionDelayMs ?? 0); + const archiveDueAt = latestTerminalAt + workspaceReaperCooldownMs; terminalWorkspaceDeadlines.set(workspace.id, reopenPending ? now().getTime() - : latestTerminalAt + workspaceReaperCooldownMs); + : compactionPending ? Math.min(compactionDueAt, archiveDueAt) : archiveDueAt); + if (compactionPending && compactionDueAt <= now().getTime()) { + try { + const outcome = await compactTerminalWorkspace(workspace, signal); + if (outcome === "compacted") result.compacted += 1; + if (outcome !== "busy") { + if (compactedTerminalAt.size >= 4096) compactedTerminalAt.delete(compactedTerminalAt.keys().next().value!); + compactedTerminalAt.set(workspace.id, latestTerminalAt); + terminalWorkspaceDeadlines.set(workspace.id, archiveDueAt); + } + } catch (error) { + if (signal?.aborted) throw error; + result.compactionFailed += 1; + logger.warn( + { err: error, executionWorkspaceId: workspace.id, sourceIssueId: workspace.sourceIssueId }, + "terminal workspace compaction failed; a later sweep retries", + ); + } + } if (workspaceReaperCooldownMs > 0 && latestTerminalAt + workspaceReaperCooldownMs > now().getTime()) { result.skippedCooldown += 1; continue; diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index eb53316d4e4..009d7264cea 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -154,14 +154,15 @@ const PUBLIC_BOOLEAN_ENV_KEYS = new Set([ "CI", "NO_COLOR", "FORCE_COLOR", "VITEST", "MALLOCNANOZONE", "CODEX_CI", "CODEX_SHELL", "CODEX_SAGE_BACKFILL_TRACKER_TAB_REUSE", "DEV", "PROD", "SSR", "PAPERCLIP_REQUIRE_SENTRY_TEST_SDK", - "PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION", + "PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_WORKSPACE_COMPACTION_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION", "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "PAPERCLIP_SECRETS_STRICT_MODE", "SERVE_UI", "PAPERCLIP_UI_DEV_MIDDLEWARE", "HEARTBEAT_SCHEDULER_ENABLED", "PAPERCLIP_ANNOUNCEMENTS_ENABLED", "GITHUB_ACTIONS", "GITHUB_REF_PROTECTED", "RUNNER_DEBUG", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG", ]); const PUBLIC_NUMBER_ENV_KEYS = new Set([ "PORT", "PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES", "PAPERCLIP_DB_BACKUP_RETENTION_DAYS", - "PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "HEARTBEAT_SCHEDULER_INTERVAL_MS", + "PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "PAPERCLIP_WORKSPACE_COMPACTION_DELAY_HOURS", + "HEARTBEAT_SCHEDULER_INTERVAL_MS", "GITHUB_RUN_ATTEMPT", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ID", "GITHUB_RETENTION_DAYS", ]); diff --git a/server/src/services/workspace-compaction.ts b/server/src/services/workspace-compaction.ts new file mode 100644 index 00000000000..e9eac8026f0 --- /dev/null +++ b/server/src/services/workspace-compaction.ts @@ -0,0 +1,232 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + +// Directory names that hold regenerable output: installed dependencies, build +// output, and tool caches. Compaction removes a directory only when its name is +// in this set, Git ignores it, and it holds no tracked file. Removing one leaves +// the worktree in the same state as a freshly created worktree, so whatever +// produced it the first time (the workspace provision command, or the agent) +// produces it again when the task resumes. +export const REGENERABLE_WORKSPACE_DIR_NAMES: ReadonlySet = new Set([ + "node_modules", + "dist", + "build", + "coverage", + "storybook-static", + "target", + ".cache", + ".next", + ".nuxt", + ".parcel-cache", + ".svelte-kit", + ".turbo", + ".vite", +]); + +// Compaction never descends into these directories. `.paperclip` holds the +// worktree's own Paperclip instance data, which is not regenerable. +const PROTECTED_WORKSPACE_DIR_NAMES: ReadonlySet = new Set([".git", ".paperclip"]); + +// Moved directories wait here before deletion. The trash sits beside the +// worktree so the move is a same-filesystem rename, which is atomic and fast. +// A branch name cannot start with ".", so the name never collides with a +// worktree directory. +export const WORKSPACE_COMPACTION_TRASH_DIR = ".paperclip-compaction-trash"; + +const MAX_WALK_DEPTH = 12; + +// Each Git read has a deadline and stops when the sweep is aborted, so a stalled +// Git process cannot hold the reaper. +const GIT_READ_TIMEOUT_MS = 60_000; + +async function runGitRead( + worktreePath: string, + args: string[], + options: { signal?: AbortSignal; input?: string; okExitCodes?: number[] } = {}, +) { + return await new Promise((resolve, reject) => { + const child = execFile( + "git", + ["-C", worktreePath, ...args], + { cwd: worktreePath, timeout: GIT_READ_TIMEOUT_MS, signal: options.signal, maxBuffer: 16 * 1024 * 1024 }, + (error, stdout) => { + const exitCode = (error as { code?: unknown } | null)?.code; + if (!error || (typeof exitCode === "number" && options.okExitCodes?.includes(exitCode))) { + resolve(stdout); + return; + } + reject(error); + }, + ); + // Git can exit before it reads stdin (or never reads it). A write to the + // closed pipe then fails with EPIPE; the exit status above already reports + // any real failure, so the stream error is not a second outcome. + child.stdin?.on("error", () => {}); + if (options.input === undefined) child.stdin?.end(); + else child.stdin?.end(options.input); + }); +} + +function splitNul(output: string) { + return output.split("\0"); +} + +function isRepositoryGitignore(source: string) { + return !path.isAbsolute(source) + && !source.split("/").includes("..") + && path.posix.basename(source) === ".gitignore"; +} + +// Keeps the candidates that Git ignores and that hold no tracked file. +// `check-ignore` answers per path, also for a path below an ignored parent. +// It exits 1 when no path is ignored, which is a normal answer. +async function filterRemovableCandidates(worktreePath: string, candidates: string[], signal?: AbortSignal) { + // With --verbose --non-matching -z, each path yields four fields: source, + // line number, pattern, and path. The source is empty for a path that no + // pattern matches. Verbose output also reports a negating pattern such as + // `!build/`, which means Git does NOT ignore the path, so only a match by a + // non-negated pattern counts. Only the repository's own `.gitignore` files + // count as evidence: a user's global excludes file or `.git/info/exclude` + // says nothing about whether this repository's directory is regenerable. + const fields = splitNul(await runGitRead( + worktreePath, + ["check-ignore", "--stdin", "-z", "--verbose", "--non-matching"], + { signal, input: candidates.map((candidate) => `${candidate}\0`).join(""), okExitCodes: [1] }, + )); + const ignored = new Set(); + for (let index = 0; index + 3 < fields.length; index += 4) { + const [source, , pattern, candidate] = fields.slice(index, index + 4); + if (source && pattern && !pattern.startsWith("!") && isRepositoryGitignore(source)) ignored.add(candidate!); + } + const ignoredCandidates = candidates.filter((candidate) => ignored.has(candidate)); + if (ignoredCandidates.length === 0) return []; + // A directory that holds a tracked file is source, even when Git ignores it. + // Literal pathspecs: a candidate is a path, never a pattern. + const tracked = splitNul(await runGitRead( + worktreePath, + ["--literal-pathspecs", "ls-files", "-z", "--", ...ignoredCandidates], + { signal }, + )) + .filter((entry) => entry.length > 0); + return ignoredCandidates.filter((candidate) => + !tracked.some((trackedPath) => trackedPath === candidate || trackedPath.startsWith(`${candidate}/`))); +} + +// Returns true only for a linked Git worktree: its `.git` entry is a file that +// points at the main repository. A main checkout has a `.git` directory, and +// compaction never touches one. +export async function isLinkedGitWorktree(worktreePath: string) { + const stat = await fs.lstat(path.join(worktreePath, ".git")).catch(() => null); + return stat?.isFile() === true; +} + +// Lists the regenerable directories in a worktree, as paths relative to the +// worktree root. Symlinks are never followed or returned. A directory with a +// regenerable name that does not qualify (for example a tracked `build/`) is +// searched in turn, so eligible directories inside it are still found. +export async function findRegenerableWorkspaceDirs( + worktreePath: string, + options: { signal?: AbortSignal } = {}, +): Promise { + const removable: string[] = []; + + async function collect(relativeDir: string, depth: number, candidates: string[]) { + options.signal?.throwIfAborted(); + if (depth > MAX_WALK_DEPTH) return; + const entries = await fs.readdir(path.join(worktreePath, relativeDir), { withFileTypes: true }) + .catch(() => []); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (PROTECTED_WORKSPACE_DIR_NAMES.has(entry.name)) continue; + const relativePath = relativeDir ? path.posix.join(relativeDir, entry.name) : entry.name; + if (REGENERABLE_WORKSPACE_DIR_NAMES.has(entry.name)) { + candidates.push(relativePath); + continue; + } + await collect(relativePath, depth + 1, candidates); + } + } + + let roots = [""]; + while (roots.length > 0) { + const candidates: string[] = []; + for (const root of roots) { + await collect(root, root ? root.split("/").length : 0, candidates); + } + if (candidates.length === 0) break; + const eligible = new Set(await filterRemovableCandidates(worktreePath, candidates, options.signal)); + removable.push(...candidates.filter((candidate) => eligible.has(candidate))); + roots = candidates.filter((candidate) => !eligible.has(candidate)); + } + return removable; +} + +export type MovedWorkspaceDir = { relativePath: string; trashPath: string }; + +export type WorkspaceCompactionMove = { + trashDir: string; + moved: MovedWorkspaceDir[]; +}; + +// Moves each directory into a fresh trash directory beside the worktree. Each +// move is one rename, so the whole step is fast enough to run while the caller +// holds the workspace lifecycle lock. A path that is no longer a real +// directory, or that sits on another filesystem, is left in place. +export async function moveWorkspaceDirsToTrash( + worktreePath: string, + relativePaths: string[], +): Promise { + const trashDir = path.join( + path.dirname(worktreePath), + WORKSPACE_COMPACTION_TRASH_DIR, + `${path.basename(worktreePath)}-${Date.now()}-${randomUUID().slice(0, 8)}`, + ); + await fs.mkdir(trashDir, { recursive: true }); + const moved: MovedWorkspaceDir[] = []; + for (const [index, relativePath] of relativePaths.entries()) { + const sourcePath = path.join(worktreePath, relativePath); + const stat = await fs.lstat(sourcePath).catch(() => null); + if (!stat?.isDirectory()) continue; + const trashPath = path.join(trashDir, String(index)); + try { + await fs.rename(sourcePath, trashPath); + moved.push({ relativePath, trashPath }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EXDEV") continue; + await restoreWorkspaceDirsFromTrash(worktreePath, { trashDir, moved }); + throw error; + } + } + return { trashDir, moved }; +} + +// Puts moved directories back. A path that something recreated in the meantime +// keeps the new copy, and the moved copy stays in the trash for deletion. +export async function restoreWorkspaceDirsFromTrash(worktreePath: string, move: WorkspaceCompactionMove) { + for (const entry of [...move.moved].reverse()) { + const targetPath = path.join(worktreePath, entry.relativePath); + const occupied = await fs.lstat(targetPath).then(() => true, () => false); + if (occupied) continue; + await fs.rename(entry.trashPath, targetPath); + } +} + +export async function deleteWorkspaceCompactionTrash(trashDir: string) { + await fs.rm(trashDir, { recursive: true, force: true }); +} + +// Deletes trash directories that a crash or restart left behind. A trash entry +// younger than `olderThanMs` may belong to a compaction in progress, so it stays. +export async function purgeStaleWorkspaceCompactionTrash(worktreeParentDir: string, olderThanMs: number) { + const trashRoot = path.join(worktreeParentDir, WORKSPACE_COMPACTION_TRASH_DIR); + const entries = await fs.readdir(trashRoot, { withFileTypes: true }).catch(() => []); + const cutoff = Date.now() - olderThanMs; + for (const entry of entries) { + const entryPath = path.join(trashRoot, entry.name); + const stat = await fs.lstat(entryPath).catch(() => null); + if (!stat || stat.mtimeMs > cutoff) continue; + await fs.rm(entryPath, { recursive: true, force: true }).catch(() => {}); + } +}