diff --git a/.github/scripts/tests/ci-profile.test.mjs b/.github/scripts/tests/ci-profile.test.mjs index b6a931b6e3c..f113d0ad87d 100644 --- a/.github/scripts/tests/ci-profile.test.mjs +++ b/.github/scripts/tests/ci-profile.test.mjs @@ -239,7 +239,14 @@ test("workflow exporter runs trusted code and keeps keys out of the PR jobs", () assert.match(workflow, /environment: ci-profiling/); assert.doesNotMatch(workflow, /head_sha|secrets: inherit|contents: write|pull-requests: write|pnpm install/); const pr = readFileSync(new URL("../../workflows/pr-trusted.yml", import.meta.url), "utf8"); - assert.doesNotMatch(pr, /HONEYCOMB.*KEY|secrets\./); + // A script filename such as docker-compose-secrets.test.mjs is not a + // GitHub secret lookup. Check expression context, including bracket access. + const secretReference = /HONEYCOMB.*KEY|\$\{\{[^}]*\bsecrets\s*(?:\.|\[)/; + assert.doesNotMatch("run: node --test scripts/docker-compose-secrets.test.mjs", secretReference); + for (const lookup of ["${{ secrets.PR_TOKEN }}", "${{ secrets['PR_TOKEN'] }}", "${{\n secrets.PR_TOKEN\n }}"]) { + assert.match(lookup, secretReference); + } + assert.doesNotMatch(pr, secretReference); assert.match(pr, /if: always\(\) && env.PAPERCLIP_CI_PROFILE == 'true'/); assert.doesNotMatch(pr, /PAPERCLIP_CI_PROFILE:.*labels/); assert.equal(pr.match(/PAPERCLIP_CI_PROFILE: \$\{\{ needs.gate.outputs.profile_ci \}\}/g).length, 9); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 6fb6f0f6643..833db6306ac 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2471,6 +2471,37 @@ fields before disabling a connection. Verify workspace ownership, active runs, Git state, and runtime-service readiness before closing a workspace. A missing URL or old workspace timestamp alone does not prove that a row is disposable. +### Reconstructable Git cache expiration + +The terminal-workspace worker can expire Paperclip-owned isolated Git worktrees +24 hours after every task in their tree is done or cancelled, independently of +PR merge or delivery status. Enable it with +`PAPERCLIP_GIT_CACHE_EXPIRATION_ENABLED=true`. It defaults to dry run; set +`PAPERCLIP_GIT_CACHE_EXPIRATION_DRY_RUN=false` after inspecting the structured +`execution_workspace.git_cache_expiration` logs. Logs report candidate and +reclaimed bytes, base removal, and skip reasons. The existing bounded lifecycle +sweep supplies scheduling and shutdown cancellation. + +Before deletion the worker fetches the exact commit into a fresh object store +using the company's normal Git identity and commits a versioned restoration +recipe to workspace metadata. Current runs, previews, pending reopening, +unresolved recovery, dirty/untracked files, unknown ignored files, unproven +ownership, and unsupported or unique Git state prevent deletion. Only the +existing repository-ignored regenerable-output allowlist can be discarded. +Deletion first renames the worktree to recorded trash; restart and reopening +finish that cleanup under the same source/lifecycle locks. + +`PAPERCLIP_GIT_BASE_EXPIRATION_ENABLED=true` additionally permits removal of an +owned isolated remote-source base after all consumers expire and all remaining +refs and files are verified reproducible. Shared project clones remain intact. +Normal authorized reopening recreates a missing source and checks out the saved +commit, even after upstream advances. Inaccessible upstream produces an explicit +retryable restoration failure. It never substitutes the default branch. + +Both enable switches default off. Rollback disables further expiration while +leaving saved recipes and restoration support intact. Recovery archives, runner +backups, non-Git workspaces, and immutable skill snapshots are unaffected. + ### Terminal workspace compaction A finished task often keeps its worktree for a long time: the reaper archives a diff --git a/doc/plans/2026-10-15-recoverable-disk-cleanup.md b/doc/plans/2026-10-15-recoverable-disk-cleanup.md index d8a11ad79fc..58b4c376937 100644 --- a/doc/plans/2026-10-15-recoverable-disk-cleanup.md +++ b/doc/plans/2026-10-15-recoverable-disk-cleanup.md @@ -35,82 +35,93 @@ product acceptance. Rollback disables deletion and retains restoration support. ## Current state -- Checkout: `/Users/dotta/.codex/worktrees/2da2/paperclip` -- Skill PR branch: `fix/skill-version-dir-retention`, based on Devin Foley's - [PR #16433](https://github.com/paperclipai/paperclip/pull/16433). User approved - building on that PR. Added safety commits to its existing branch, preserving - Devin’s original commit; latest published revision before this fix is `6385a28ce7e0ee24bf0bc5e1dc088e7f4abb9508`. - User also authorized merging every PR from this effort after checks pass. -- Earlier proof/implementation preserved at `da754ff744` on - `codex/recoverable-skill-cache`; do not ship its competing worker. -- Staging: `paperclip.staging.paperclip.app`, stack `stack-pool-50531a9d17dd`. -- Staging read-only recheck at 16:07 UTC: 71% disk used, 103,676,784,640 bytes - used, 43,621,777,408 available. Serving `7ec6fd09725fdcd771425e31ce900f8db9aae07d`. - This task has not deployed or deleted staging data. -- A temporary Railway key named `paperclip-cleanup-acceptance-2026-10-15` is - registered under the prior explicit approval. Revoke it and remove both local - copies at acceptance completion (paths retained in private task evidence). +- User authorized merging every PR from this effort after checks pass. +- Skill [PR #16433](https://github.com/paperclipai/paperclip/pull/16433) merged + after exact-head CI, security review, and Greptile 5/5 passed. Reviewed head: + `afc159cf39a1228608c1f864648ab359a57bf357`; merge: + `86c965ce7ab1499864f74b8dd6ed0d2d8db15197`. Devin's original implementation + and authorship are retained. The earlier prototype on `codex/recoverable-skill-cache` + must not ship as a competing worker. +- Git [PR #16457](https://github.com/paperclipai/paperclip/pull/16457) is open on + `codex/recoverable-git-cache`, rebased onto that merge. Initial review head: + `bb3984d3fd13e7a5604e9d90c7d58748d4b50708`. CI and review are pending. +- Git evidence: 17 real database/filesystem cases passed in 48.04s, including + expiration/reopening at A after upstream B, subsequent commits, interrupted + trash deletion, concurrent reopening, unique files/Git state, live consumers, + upstream failure, company boundaries, shared bases, and nested private sources. + Three final guard cases passed; 328 existing controls passed in 238.69s. + Evidence: `/tmp/paperclip-git-expiration-tests4.log`, + `/tmp/paperclip-git-final-guards.log`, `/tmp/paperclip-git-existing-controls.log`. +- Skill evidence: 21 final focused cases passed in 29.99s and 35 affected + skill/heartbeat controls passed. Streaming discovery holds directory cursors + across bounded pages; a regression counts actual reads. Preparation/pinning + and eviction share company locks; filesystem inspection/deletion do not hold + the preparation lock. Immutable snapshots remain in the database. +- Full local typecheck/build passed for both slices. The full test attempt ran + for 3,547.58s and did not pass: startup IPC capacity failures, mixed-revision + discovery tests, two unrelated controls, and module failures after the checkout + disappeared. It is not evidence of one clean SHA. Exact-head CI supersedes + the mixed-revision run for the merged skill PR. Focused final-code retries + classified the remaining failures: all 103 tests in the affected final-code + retry passed in 42.25s. Do not repeat an hour-long loop without cause. +- The checkout unexpectedly disappeared at about 17:22 UTC after all code was + pushed. Restored the same branch/path from Git, reinstalled frozen dependencies + in 24.8s, and rebuilt successfully. Managed attachment recovery was unavailable + because the app could not verify ownership. All retained evidence is in `/tmp`. +- Staging target: `paperclip.staging.paperclip.app`, `stack-pool-50531a9d17dd`. + One deployment request owns this task's rollout: + `38a0a286-7a97-4cd5-b58e-eabdaa4d38bd`, source merge `86c965ce7a…`. + First prepare failed with `direct_migrator_artifact_invalid` while the exact + source artifact was still publishing. After its publisher passed, reran that + failed prepare and resumed the same request. No campaign duplication. +- Latest read-only staging baseline at 17:26 UTC: serving + `03f22f4cb9be9a34906bda0e4906262dc359e9ce`, disk 73%, 106,903,482,368 bytes + used and 40,395,079,680 available. This task has not deleted staging data. + Skill cleanup is configured enabled with dry-run true; Git cleanup is not enabled. - Existing unresolved staging runs have missing skill-version pins. The approved - conservative policy skips that company; do not clear recovery or weaken the - guard to obtain reclamation. Qualify the skill slice with a disposable pinned - fixture and report the legacy-data gap separately. -- Integrated one worker, cross-process preparation/pin locks, separate last-use - metadata, legacy observation grace, unresolved-recovery protection, dry run, - bounded scans/continuations, and deletion outside the preparation lock. -- First integrated run: 22 focused tests passed; broader controls passed 100/101. - Removed an unnecessary explicit-deletion lock exposed by the rollback fixture; - that control passed on rerun (12.21s). The original failure remains in - `/tmp/paperclip-skill-integration-controls.log`. -- Full typecheck initially found a plugin dependency API mismatch from the - unlocked bootstrap install. Current master supplied the refreshed lockfile; - frozen install completed in 18.7s and full typecheck passed in 59.82s. -- Review at `5eb505ff5d` was 3/5: unsafe skill roots blocked the company sweep, - and recursive byte counts held the preparation lock. Both fixed locally; - 18 affected tests passed in 24.94s, including preparation during a paused file - walk and fresh pin revalidation. Server typecheck passed after those fixes. -- Full tests were stopped after approximately four minutes to address review - findings before final qualification; no full-suite pass claimed. Full build at - `d153929d70` passed. CI at that SHA passed all but the empty-runtime suite and - Greptile (4/5). The remaining review finding was repeated company-wide pin - queries; batching reads twice per sweep phase fixes it and has a query-count - regression. The empty-selection fast path now returns before locking. -- Final affected skill/heartbeat controls: 35 tests passed in 29.17s, log - `/tmp/paperclip-skill-final-controls.log`. A prior local attempt could not start - Postgres because macOS shared-memory slots were full; retry succeeded after a - slot became available, without changing global limits or unrelated services. - Full typecheck on `666c60c859` passed in 32.98s. Full tests are still running; - initial native-session cases hit the same infrastructure limit. Removed 19 - abandoned 56-byte IPC headers only after verifying zero attachments and dead - creator/last-user PIDs. Live databases and their files were untouched. -- Review round four accepted batched reads but found unbounded per-directory - discovery stats. Discovery now uses directory-entry types and applies the cursor - and budget before per-path inspection, including empty-root budget accounting. - Twenty affected tests passed in 22.05s, including filesystem-call counts across - continuation pages (`/tmp/paperclip-skill-discovery-budget.log`). Final review - and full checks remain. Review round five found an exhausted-root continuation - edge case; the cursor now skips completed roots before budgeting discovery. - Twenty-one affected tests passed in 27.98s, including a page-boundary regression - (`/tmp/paperclip-skill-continuation.log`). The server CI shard stopped with - "The operation was canceled" and no failing assertion; investigate its timeout. -- Concurrent Git work is on `codex/recoverable-git-cache` and remains uncommitted. - Actual runtime/service reopening now restores A after deleting the worktree, - branch, and base while upstream is at B. A subsequent commit survives another - reopen. The real repository/DB experiment took 6.831s; evidence is - `/tmp/paperclip-git-recovery-candidate.json`. Expiration is not implemented yet. -- First Git baseline used a real bare remote, runtime workspace creation, a - database row, and ordinary reopening: missing worktree restores A while its - branch exists; missing branch restores upstream B; missing base fails. Fixture - script `/tmp/paperclip-git-recovery-baseline.mts`, inspected output - `/tmp/paperclip-git-recovery-baseline.json`, retained Git fixture - `/tmp/paperclip-git-baseline-sbJWbo`. The test DB was cleaned up. -- Greptile cleared `0da71b2ca2` (all its findings resolved). Security review then - identified repeated full-directory enumeration. Discovery now retains bounded - directory iterators across continuations, counts root entries in its budget, - closes abandoned iterators, and bounds company discovery. Twenty-one focused - cases passed in 29.99s (`/tmp/paperclip-skill-streaming-final.log`); server - typecheck passed. A new regression inspects actual directory reads/opener reuse. -- No deployed artifact or staging acceptance yet. Next: publish review fixes, - qualify final skill SHA, then perform dry-run rollout and measured restoration. + conservative policy skips that company. Do not clear recovery or fabricate pins + to obtain savings. A disposable fixture must distinguish worker proof from the + ordinary product restoration journey and disclose this legacy-data limitation. +- UI fixture created through Studio: skill `06a02b07-ee23-421c-afd1-5aa4b1dad423`, + slug `cache-restoration-acceptance-20261015`, initial saved marker + `SAVED-SKILL-V1-20261015`. No test run yet. Clean up only this task's fixtures. +- Live Git inventory: 152 active Git workspaces, 150 with runtime ownership and + 37 with isolated remote sources. Other owned task worktrees use shared managed + bases. Support both layouts while preserving shared bases. +- Temporary Railway key `paperclip-cleanup-acceptance-2026-10-15` remains + registered under explicit user approval. Revoke it and remove both local copies + when access is no longer needed; private paths are in retained task evidence. +- Git review found two second-cycle bugs. The worker now clears the consumed + reopen fence before expiration dispatch; successful reopening marks the + recipe restored so a later missing worktree can reattach its newer branch. Eighteen + cases passed; an added symlink fixture initially broke its own relative Git + commondir. After retaining a valid absolute commondir, that safety case passed + in 10.63s. The new guard rejects symlinked administration storage before deletion. + A second review caught the normal preparation refresh moving A to B after + recipe retirement. Keeping a restored recipe preserves the refresh guard while + accepting subsequent branch commits. Both joined lifecycle tests now pass in + 35.99s, including normal runtime preparation after reopening and another + missing-worktree rebuild after new work. The premature partial-edit test run + is excluded; `/tmp/paperclip-git-resumed-preparation2.log` is the completed run. + Full typecheck/build at `465e57ad52` passed; the guard change has its own server + typecheck. A CI policy test also mistook a script filename for a + GitHub secret expression; scope its check to expressions and retain explicit + dot/bracket/multiline secret controls. +- Remaining: address Git review, merge after all gates pass, verify exact serving + SHAs, dry-run both slices, measure actual reclamation, use restored skill and + reopened workspace through the product, inspect persisted results and protected + data, record final evidence, and revoke temporary access. No live acceptance yet. +- Latest branch-adoption review fix keeps the retained recipe's branch synchronized + with the row inside the existing reconciliation transaction. The joined restore, + rename, adoption, and normal-preparation case passed in 16.67s; 11 existing + reconciliation controls passed in 21.83s. Full typecheck/build passed for this + change. Evidence: `/tmp/paperclip-git-branch-adoption.log`, + `/tmp/paperclip-git-reconcile-controls.log`, and + `/tmp/paperclip-git-adoption-{typecheck,build}.log`. +- Skill deployment artifacts finished, but its deploy stage was rejected at + 17:50 UTC because another campaign already held the target. No competing + campaign was created. Inspect that operation before rerunning the same request. + UI fixture v2 is saved; neither staging fixture eviction nor restoration has run. ## Feedback-loop evidence diff --git a/maps/agent-access-map/workspaces.md b/maps/agent-access-map/workspaces.md index afcfaf3f9ca..354ea195f1d 100644 --- a/maps/agent-access-map/workspaces.md +++ b/maps/agent-access-map/workspaces.md @@ -57,6 +57,14 @@ are revalidated for reuse, cleanup and reopening. A failed save removes only the owned task worktree. The [isolated-source service](../../server/src/services/isolated-remote-workspace-source.ts) and its local regression tests establish source and lifecycle boundaries; live agent/provider qualification remains separate. +Reconstructable Git expiration adds no agent endpoint or permission. It records +company/workspace/repository identity and an exact commit before removing owned +cache files. Existing task/workspace reopening retains current company, source, +and trust checks. Missing upstream access returns a retryable error; it does not +choose a newer commit or grant access to another company's source. Shared project +bases and unresolved recovery remain protected. Local lifecycle evidence is in +[expiration tests](../../server/src/__tests__/git-workspace-expiration.test.ts). + Writable aliases must also be compatible with current and retained physical source projects' trust boundaries. Incompatible choices are rejected before dispatch; a projectless alias cannot bypass required sandbox containment. diff --git a/maps/feature-map/workspaces.md b/maps/feature-map/workspaces.md index 39a79412f21..4bda0d21c3c 100644 --- a/maps/feature-map/workspaces.md +++ b/maps/feature-map/workspaces.md @@ -57,6 +57,15 @@ and [preparation tests](../../server/src/services/heartbeat/workspace-preparatio exercise the real Git isolation and lifecycle guards. They do not qualify live remote credentials or a provider deployment. +For reconstructable Git expiration, finish a disposable task tree, run the +internal lifecycle sweep after its 24-hour grace, and inspect candidate/reclaimed +bytes. Advance its upstream first; use the ordinary task/workspace reopen control +and verify the original commit, file contents, and subsequent persisted work. +Current runs, previews, recovery, unique files and shared base clones must stay. +The [expiration tests](../../server/src/__tests__/git-workspace-expiration.test.ts) +exercise real local Git and database paths, interruption and concurrent reopening; +cloud credentials and the deployed user journey require separate live evidence. + For task workspace selection, use the authorized task workspace API to choose a task directory, project source, or existing workspace. Verify that a change made during a run stays pending until the next normal admission. Submit another diff --git a/server/src/__tests__/git-workspace-expiration.test.ts b/server/src/__tests__/git-workspace-expiration.test.ts new file mode 100644 index 00000000000..b0a3d7eb8ae --- /dev/null +++ b/server/src/__tests__/git-workspace-expiration.test.ts @@ -0,0 +1,297 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { companies, projects, issues, executionWorkspaces, agents, heartbeatRuns, nativeRunFinalizations, workspaceRuntimeServices, createDb } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; +import { materializeManagedProjectWorkspace } from "../services/managed-repository-checkout.js"; +import { materializeIsolatedRemoteWorkspaceSource } from "../services/isolated-remote-workspace-source.js"; +import { realizeExecutionWorkspace, ensurePersistedExecutionWorkspaceAvailable } from "../services/workspace-runtime.js"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { expireReconstructableGitWorkspace } from "../services/git-workspace-expiration.js"; + +const gitEnv = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test", GIT_TERMINAL_PROMPT: "0" }; +const git = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, env: gitEnv, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +const exists = (target: string) => fs.lstat(target).then(() => true, () => false); + +const support = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = support.supported ? describe : describe.skip; +describeEmbeddedPostgres("reconstructable Git expiration", () => { + let db: ReturnType; + let database: Awaited>; + let root: string; + const previousHome = process.env.PAPERCLIP_HOME; + const previousInstance = process.env.PAPERCLIP_INSTANCE_ID; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("git-cache-expiration-"); + db = createDb(database.connectionString); + root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-expiration-")); + process.env.PAPERCLIP_HOME = path.join(root, "home"); + process.env.PAPERCLIP_INSTANCE_ID = "default"; + }); + afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); }); + afterAll(async () => { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = previousInstance; + await db.$client.end({ timeout: 1 }); await database.cleanup(); await fs.rm(root, { recursive: true, force: true }); + }); + + async function fixture(managed = false, nested = false) { + const companyId = randomUUID(), issueId = randomUUID(), workspaceId = randomUUID(); + const location = path.join(root, companyId), source = path.join(location, "author"), remote = path.join(location, "remote.git"); + await fs.mkdir(source, { recursive: true }); + git(source, "init", "--initial-branch=main"); + await fs.writeFile(path.join(source, "marker.txt"), "commit A\n"); + await fs.writeFile(path.join(source, ".gitignore"), "node_modules/\nprivate-notes/\n"); + git(source, "add", "."); git(source, "commit", "-m", "A"); + const commitA = git(source, "rev-parse", "HEAD"); + git(location, "clone", "--bare", source, remote); git(source, "remote", "add", "origin", remote); + await db.insert(companies).values({ id: companyId, name: "Git expiration fixture", issuePrefix: `GC${companyId.slice(0, 6)}` }); + await db.insert(issues).values({ id: issueId, companyId, title: "Finished task", status: "done", completedAt: new Date(Date.now() - 48 * 3_600_000) }); + const repoUrl = pathToFileURL(remote).href; + const projectId = managed ? randomUUID() : null; + if (projectId) await db.insert(projects).values({ id: projectId, companyId, name: "Fixture project" }); + const baseCwd = managed + ? (await materializeManagedProjectWorkspace(resolveManagedProjectWorkspaceDir({ companyId, projectId: projectId!, repoName: "remote" }), { repoUrl })).cwd + : await materializeIsolatedRemoteWorkspaceSource({ companyId, issueId, repoUrl, resolveGitAuth: async () => null }); + const base = { baseCwd, source: "task_session" as const, projectId: null, workspaceId: null, repoUrl, repoRef: "origin/main" }; + const workspace = await realizeExecutionWorkspace({ base, issue: { id: issueId, identifier: "GC-1", title: "Fixture" }, agent: { id: null, name: "Fixture", companyId }, + config: { workspaceStrategy: { type: "git_worktree", branchTemplate: "fixture-task", worktreeParentDir: nested ? path.join(baseCwd, ".paperclip", "worktrees") : path.join(location, "worktrees"), baseRef: "origin/main" } }, skipHostProvision: true }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, projectId, sourceIssueId: issueId, mode: "isolated_workspace", strategyType: "git_worktree", name: "Fixture", status: "active", + providerType: "local_fs", cwd: workspace.cwd, providerRef: workspace.cwd, repoUrl, baseRef: "origin/main", branchName: workspace.branchName, + metadata: { createdByRuntime: workspace.branchCreatedByRuntime, gitBranchOwnershipVersion: 1, baseRefSha: workspace.baseRefSha, + ...(managed ? {} : { isolatedRemoteSource: { schema: "paperclip.isolated-remote-source.v1", companyId, issueId, repoUrl } }) } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + const row = () => db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)).then(rows => rows[0]!); + const expire = async (options: Partial[2]> = {}) => + expireReconstructableGitWorkspace(db, await row(), { dryRun: false, expireBase: true, now: new Date(), ...options }); + const reopen = (company = companyId) => executionWorkspaceService(db).reopenClosedIsolatedExecutionWorkspaceForIssue({ workspaceId, + issue: { id: issueId, companyId: company, projectId: null }, actor: { actorType: "user", agentId: null }, + authorizationActor: { type: "board", source: "local_implicit", isInstanceAdmin: true } }); + const advance = async () => { + await fs.writeFile(path.join(source, "marker.txt"), "commit B\n"); git(source, "add", "."); git(source, "commit", "-m", "B"); git(source, "push", "origin", "main"); + return git(source, "rev-parse", "HEAD"); + }; + return { companyId, issueId, workspaceId, cwd: workspace.cwd, baseCwd, base, remote, source, commitA, row, expire, reopen, advance }; + } + + it("expires through the lifecycle worker and restores the recorded commit after upstream advances", async () => { + const f = await fixture(); + await fs.mkdir(path.join(f.cwd, "node_modules")); await fs.writeFile(path.join(f.cwd, "node_modules", "cache"), "regenerable"); + await fs.symlink("cache", path.join(f.cwd, "node_modules", "cache-link")); + expect(await f.expire({ dryRun: true })).toMatchObject({ candidate: true, expired: false, reclaimedBytes: 0, skipReason: null }); + expect(await exists(f.cwd)).toBe(true); expect((await f.row()).metadata?.gitCacheRestoration).toBeUndefined(); + await f.advance(); + const outcome = await executionWorkspaceService(db, { gitCacheExpirationEnabled: true, gitBaseExpirationEnabled: true }).sweepTerminalWorkspaces(); + expect(outcome.archived).toBe(1); + expect(await exists(f.cwd)).toBe(false); expect(await exists(f.baseCwd)).toBe(false); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + expect(await fs.readFile(path.join(f.cwd, "marker.txt"), "utf8")).toBe("commit A\n"); + // The next heartbeat uses this runtime path before handing files to the agent. + await ensurePersistedExecutionWorkspaceAvailable({ db, base: f.base, + workspace: (await executionWorkspaceService(db).getStoredById(f.companyId, f.workspaceId))!, + issue: { id: f.issueId, identifier: "GC-1", title: "Resumed task" }, + agent: { id: null, name: "Fixture", companyId: f.companyId }, skipHostProvision: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + await fs.writeFile(path.join(f.cwd, "continued.txt"), "continued\n"); git(f.cwd, "add", "."); git(f.cwd, "commit", "-m", "Continued"); + const continued = git(f.cwd, "rev-parse", "HEAD"); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: false }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(continued); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA, restoredAt: expect.any(String) }); + git(f.baseCwd, "worktree", "remove", f.cwd); + await db.update(executionWorkspaces).set({ status: "archived" }).where(eq(executionWorkspaces.id, f.workspaceId)); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(continued); + }); + + it("expires again after a reopened task consumes its fence and finishes", async () => { + const f = await fixture(); + expect((await f.expire()).expired).toBe(true); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + await db.update(issues).set({ status: "in_progress", completedAt: null }).where(eq(issues.id, f.issueId)); + const worker = executionWorkspaceService(db, { gitCacheExpirationEnabled: true, gitBaseExpirationEnabled: true }); + await worker.sweepTerminalWorkspaces(); + expect((await f.row()).metadata?.reopenPendingConsumption).not.toBe(true); + await fs.writeFile(path.join(f.cwd, "continued.txt"), "second task cycle\n"); + git(f.cwd, "add", "."); git(f.cwd, "commit", "-m", "Second task cycle"); + git(f.cwd, "push", "origin", "HEAD:refs/heads/continued"); + const continued = git(f.cwd, "rev-parse", "HEAD"); + await db.update(issues).set({ status: "done", completedAt: new Date(Date.now() - 48 * 3_600_000) }).where(eq(issues.id, f.issueId)); + expect((await worker.sweepTerminalWorkspaces()).archived).toBe(1); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: continued }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(continued); + }); + + it("prepares a restored workspace after adopting its renamed branch", async () => { + const f = await fixture(); + await f.advance(); + expect((await f.expire()).expired).toBe(true); + expect(await f.reopen()).toMatchObject({ ok: true }); + git(f.cwd, "branch", "-m", "renamed-task"); + await db.update(executionWorkspaces).set({ status: "idle" }).where(eq(executionWorkspaces.id, f.workspaceId)); + const svc = executionWorkspaceService(db); + await svc.reconcileExecutionWorkspaceBranch(f.workspaceId, { + mode: "forward", actor: { actorType: "user", actorId: "local-board", agentId: null, runId: null }, + }); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ + branchName: "renamed-task", commit: f.commitA, restoredAt: expect.any(String), + }); + await ensurePersistedExecutionWorkspaceAvailable({ db, base: f.base, + workspace: (await svc.getStoredById(f.companyId, f.workspaceId))!, + issue: { id: f.issueId, identifier: "GC-1", title: "Resumed task" }, + agent: { id: null, name: "Fixture", companyId: f.companyId }, skipHostProvision: true }); + expect(git(f.cwd, "branch", "--show-current")).toBe("renamed-task"); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + + it("preserves worktrees whose Git administration parent is a symlink", async () => { + const f = await fixture(); + const adminParent = path.join(f.baseCwd, ".git", "worktrees"); + const external = path.join(root, `external-admin-${randomUUID()}`); + const admin = git(f.cwd, "rev-parse", "--absolute-git-dir"); + await fs.writeFile(path.join(admin, "commondir"), `${path.join(f.baseCwd, ".git")}\n`); + await fs.rename(adminParent, external); + await fs.symlink(external, adminParent); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + expect(await f.expire()).toMatchObject({ expired: false, skipReason: "unproven_worktree_admin" }); + expect(await exists(f.cwd)).toBe(true); + expect(await fs.readdir(external)).not.toEqual([]); + expect((await f.row()).metadata?.gitCacheRestoration).toBeUndefined(); + }); + + it.each(["dirty", "untracked", "ignored", "local_commit", "stash", "unknown_git_file"])("preserves %s unique state", async kind => { + const f = await fixture(); + if (kind === "dirty") await fs.writeFile(path.join(f.cwd, "marker.txt"), "unique changes"); + if (kind === "untracked") await fs.writeFile(path.join(f.cwd, "notes.txt"), "unique notes"); + if (kind === "ignored") { await fs.mkdir(path.join(f.cwd, "private-notes")); await fs.writeFile(path.join(f.cwd, "private-notes", "note"), "unique ignored"); } + if (kind === "local_commit") { await fs.writeFile(path.join(f.cwd, "new.txt"), "unique commit"); git(f.cwd, "add", "."); git(f.cwd, "commit", "-m", "Unpushed"); } + if (kind === "stash") { await fs.writeFile(path.join(f.cwd, "marker.txt"), "stash"); git(f.cwd, "stash"); } + if (kind === "unknown_git_file") await fs.writeFile(path.join(git(f.cwd, "rev-parse", "--absolute-git-dir"), "unique-note"), "unique Git data"); + expect((await f.expire()).expired).toBe(false); + expect(await exists(f.cwd)).toBe(true); expect(await exists(f.baseCwd)).toBe(true); + }); + + it("preserves live descendants and pending reopening", async () => { + const f = await fixture(); + await db.insert(issues).values({ companyId: f.companyId, title: "Active child", status: "in_progress", parentId: f.issueId }); + expect((await f.expire()).expired).toBe(false); + await db.delete(issues).where(eq(issues.parentId, f.issueId)); + const row = await f.row(); + await db.update(executionWorkspaces).set({ metadata: { ...row.metadata, reopenPendingConsumption: true } }).where(eq(executionWorkspaces.id, f.workspaceId)); + expect((await f.expire()).expired).toBe(false); + expect(await exists(f.cwd)).toBe(true); + }); + + it("retries after interruption with the restoration recipe already committed", async () => { + const f = await fixture(); + expect((await f.expire({ beforeDelete: async () => { throw new Error("interrupted_fixture"); } })).expired).toBe(false); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA }); + expect(await exists(f.cwd)).toBe(true); + expect((await f.expire()).expired).toBe(true); + expect(await f.reopen()).toMatchObject({ ok: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + + it("fails restoration explicitly on upstream failure and rejects a different company", async () => { + const f = await fixture(); + expect((await f.expire()).baseExpired).toBe(true); + await fs.rename(f.remote, `${f.remote}.offline`); + expect(await f.reopen()).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry." }); + expect(await f.reopen(randomUUID())).toMatchObject({ ok: false, code: "not_reopenable" }); + expect(await exists(f.cwd)).toBe(false); + await fs.rename(`${f.remote}.offline`, f.remote); + expect(await f.reopen()).toMatchObject({ ok: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + it("serializes concurrent reopening after deletion and publishes one transition", async () => { + const f = await fixture(); + let release!: () => void, entered!: () => void; + const paused = new Promise(resolve => { release = resolve; }); + const ready = new Promise(resolve => { entered = resolve; }); + const expiration = f.expire({ beforeDelete: async () => { entered(); await paused; } }); + await ready; + const first = f.reopen(), second = f.reopen(); + release(); + expect((await expiration).expired).toBe(true); + const results = await Promise.all([first, second]); + expect(results, JSON.stringify(results)).toEqual(expect.arrayContaining([expect.objectContaining({ ok: true, reopened: true }), expect.objectContaining({ ok: true, reopened: false })])); + expect(results.filter(result => result.ok && result.reopened)).toHaveLength(1); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + + it("recovers interrupted trash removal before reopening", async () => { + const f = await fixture(); + const originalRemove = fs.rm.bind(fs); + const remove = vi.spyOn(fs, "rm").mockImplementation(async (target, options) => { + if (String(target).includes(".expired-git-worktree-")) throw new Error("interrupted_trash_fixture"); + return originalRemove(target, options); + }); + expect((await f.expire()).skipReason).toBe("interrupted_trash_fixture"); + expect(await exists(f.cwd)).toBe(false); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA }); + remove.mockRestore(); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + + it("preserves scheduled runs, unresolved recovery, and live previews", async () => { + const f = await fixture(); + const agentId = randomUUID(), runId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId: f.companyId, name: "Fixture", role: "engineer", adapterType: "process" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId, nativeIssueId: f.issueId, status: "scheduled_retry" }); + expect((await f.expire()).expired).toBe(false); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId: f.companyId, issueId: f.issueId, phase: "terminal_failure", recoveryState: "blocked" }); + expect((await f.expire()).expired).toBe(false); + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + await db.insert(workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId, executionWorkspaceId: f.workspaceId, + scopeType: "execution_workspace", serviceName: "Preview", status: "running", lifecycle: "persistent", provider: "local" }); + expect((await f.expire()).expired).toBe(false); + expect(await exists(f.cwd)).toBe(true); + }); + + it("skips deletion when authenticated upstream verification fails", async () => { + const f = await fixture(); + await fs.rename(f.remote, `${f.remote}.offline`); + expect((await f.expire()).expired).toBe(false); + expect(await exists(f.cwd)).toBe(true); + expect((await f.row()).metadata?.gitCacheRestoration).toBeUndefined(); + }); + + it("keeps an unused base containing custom Git files", async () => { + const f = await fixture(); + const note = path.join(f.baseCwd, ".git", "private-note"); + await fs.writeFile(note, "unique base metadata"); + const outcome = await f.expire(); + expect(outcome).toMatchObject({ expired: true, baseExpired: false, skipReason: "unknown_base_git_files" }); + expect(await fs.readFile(note, "utf8")).toBe("unique base metadata"); + expect(await f.reopen()).toMatchObject({ ok: true }); + }); + + it("expires owned task worktrees while preserving their shared managed base", async () => { + const f = await fixture(true, true); + await f.advance(); + expect(await f.expire()).toMatchObject({ expired: true, baseExpired: false, skipReason: "shared_base_preserved" }); + expect(await exists(f.baseCwd)).toBe(true); + expect(await f.reopen()).toMatchObject({ ok: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + + it("expires an isolated base whose worktree was nested inside it", async () => { + const f = await fixture(false, true); + expect(await f.expire()).toMatchObject({ expired: true, baseExpired: true }); + expect(await exists(f.baseCwd)).toBe(false); + expect(await f.reopen()).toMatchObject({ ok: true }); + expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); + }); + +}); diff --git a/server/src/config.ts b/server/src/config.ts index 8ea462b8fbe..4d752fbdb01 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -79,6 +79,9 @@ export interface Config { databaseBackupIntervalMinutes: number; databaseBackupRetentionDays: number; databaseBackupDir: string; + gitCacheExpirationEnabled: boolean; + gitBaseExpirationEnabled: boolean; + gitCacheExpirationDryRun: boolean; workspaceReaperCooldownDays: number; workspaceCompactionDelayHours: number | null; skillVersionRetentionEnabled: boolean; @@ -375,6 +378,9 @@ export function loadConfig(): Config { databaseBackupIntervalMinutes, databaseBackupRetentionDays, databaseBackupDir, + gitCacheExpirationEnabled: process.env.PAPERCLIP_GIT_CACHE_EXPIRATION_ENABLED === "true", + gitBaseExpirationEnabled: process.env.PAPERCLIP_GIT_BASE_EXPIRATION_ENABLED === "true", + gitCacheExpirationDryRun: process.env.PAPERCLIP_GIT_CACHE_EXPIRATION_DRY_RUN !== "false", workspaceReaperCooldownDays, workspaceCompactionDelayHours, skillVersionRetentionEnabled: process.env.PAPERCLIP_SKILL_VERSION_RETENTION_ENABLED !== "false", diff --git a/server/src/index.ts b/server/src/index.ts index 3221a851ffc..b76a63fb107 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1412,6 +1412,9 @@ async function startServerWithDatabaseTeardown( wakeup: heartbeat.wakeup, }); const terminalWorkspaces = executionWorkspaceService(db as any, { + gitCacheExpirationEnabled: config.gitCacheExpirationEnabled, + gitBaseExpirationEnabled: config.gitBaseExpirationEnabled, + gitCacheExpirationDryRun: config.gitCacheExpirationDryRun, workspaceReaperCooldownDays: config.workspaceReaperCooldownDays, workspaceCompactionDelayMs: config.workspaceCompactionDelayHours === null ? null diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index d3a80e3aa94..8463c768eae 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -1,7 +1,9 @@ import { measureRunTransaction } from "./execution-transaction-timing.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; import { assertTaskWorkspaceAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-source-access.js"; -import { verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; +import { verifiedRecordedIsolatedRemoteSource, withIsolatedRepositorySourceLock } from "./isolated-remote-workspace-source.js"; +import { expireReconstructableGitWorkspace, hasPendingGitExpiration } from "./git-workspace-expiration.js"; +import { readGitCacheRestoration, restoreGitCacheBase, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; import { parseObject } from "../adapters/utils.js"; import { resolveCoreTrustPreset } from "./trust-preset-resolver.js"; import { CONTROLLER_PROVISION_COPY_ONLY, workspaceRequiresCopyOnlyProvisioning } from "./workspace-host-provision.js"; @@ -286,6 +288,9 @@ export type ReopenClosedIsolatedExecutionWorkspaceResult = | { ok: false; code: "not_reopenable" | "rebuild_failed"; message: string }; export type ExecutionWorkspaceServiceOptions = { + gitCacheExpirationEnabled?: boolean; + gitBaseExpirationEnabled?: boolean; + gitCacheExpirationDryRun?: boolean; terminalWorkspaceInspectionBackoffMs?: number; resolvePullRequestDetails?: PullRequestMergeDetailsResolver; now?: () => Date; @@ -3132,9 +3137,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic try { if (options.compact !== false) await purgeDeferredCompactionTrash(signal); const baseCandidateFilter = and( - inArray(executionWorkspaces.status, ["active", "idle", "in_review"]), - isNull(executionWorkspaces.closedAt), sql`${executionWorkspaces.sourceIssueId} IS NOT NULL`, + or(and(inArray(executionWorkspaces.status, ["active", "idle", "in_review"]), isNull(executionWorkspaces.closedAt)), + opts.gitCacheExpirationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]), + sql`${executionWorkspaces.metadata}->'gitCacheExpiration'->>'completed' = 'false'`) : undefined), ); // Continue the scan after the previous sweep's last row. The keyset // predicate uses the same (updatedAt, id) order as the query, so each @@ -3222,6 +3228,21 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic result.skippedNonTerminalTree += 1; continue; } + if (opts.gitCacheExpirationEnabled && (hasPendingGitExpiration(workspace) + || (workspace.mode === "isolated_workspace" && workspace.strategyType === "git_worktree"))) { + const expiration = await expireReconstructableGitWorkspace(db, workspace, { + dryRun: opts.gitCacheExpirationDryRun ?? false, expireBase: opts.gitBaseExpirationEnabled ?? false, + now: now(), signal, beforeDelete: opts.beforeTerminalWorkspaceCleanup, + }); + terminalWorkspaceDeadlines.set(workspace.id, now().getTime() + 60 * 60 * 1000); + if (expiration.candidate) result.eligible += 1; + if (expiration.expired) result.archived += 1; + logger.info({ event: "execution_workspace.git_cache_expiration", executionWorkspaceId: workspace.id, + companyId: workspace.companyId, dryRun: opts.gitCacheExpirationDryRun ?? false, ...expiration }, + "Git workspace cache expiration inspected"); + // A failed reconstruction proof must not fall through to legacy deletion. + continue; + } const latestTerminalAt = Math.max(...issueTree.map(issue => issueTerminalTimestamp(issue)!.getTime())); // Compaction runs once per terminal cycle, after its own delay and // before the archive cooldown, so a workspace the reaper cannot archive @@ -3632,13 +3653,29 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic eq(issues.executionWorkspaceId, input.workspaceId), await issueReadSqlCondition(db, input.authorizationActor), )); - const snapshot = boundSnapshot ? await db.select().from(executionWorkspaces).where(and( + const authorizedSnapshot = boundSnapshot ? await db.select().from(executionWorkspaces).where(and( eq(executionWorkspaces.id, input.workspaceId), eq(executionWorkspaces.companyId, issue.companyId), eq(executionWorkspaces.mode, "isolated_workspace"), await executionWorkspaceReadSqlCondition(db, input.authorizationActor), )).then(rows => rows[0] ?? null) : null; - if (!snapshot) return { ok: false, code: "not_reopenable", message: "Execution workspace is not reopenable" }; + if (!authorizedSnapshot) return { ok: false, code: "not_reopenable", message: "Execution workspace is not reopenable" }; + await assertTaskWorkspaceAccess(db, input.authorizationActor, issue.companyId, authorizedSnapshot.id); + const sourceIssueId = authorizedSnapshot.sourceIssueId; + const reopen = async (): Promise => { + const freshSnapshot = await db.select().from(executionWorkspaces).where(and( + eq(executionWorkspaces.id, input.workspaceId), eq(executionWorkspaces.companyId, issue.companyId), + eq(executionWorkspaces.mode, "isolated_workspace"), + await executionWorkspaceReadSqlCondition(db, input.authorizationActor), + )).then(rows => rows[0] ?? null); + if (!freshSnapshot || freshSnapshot.sourceIssueId !== sourceIssueId) { + return { ok: false, code: "not_reopenable", message: "Execution workspace source changed during reopen" }; + } + const snapshot = freshSnapshot; await assertTaskWorkspaceAccess(db, input.authorizationActor, issue.companyId, snapshot.id); + let recipe: ReturnType; + try { recipe = readGitCacheRestoration(snapshot); } + catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; } + const gitAuth = createGitRemoteAuthProvider(db, snapshot.companyId, { issueId: snapshot.sourceIssueId ?? issue.id }); const [ { ensurePersistedExecutionWorkspaceAvailable }, { workspaceOperationService }, @@ -3666,8 +3703,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic .then((rows) => parseProjectExecutionWorkspacePolicy(rows[0]?.executionWorkspacePolicy)), ]); let resolvedBaseCwd: string | null; - try { resolvedBaseCwd = await verifiedRecordedIsolatedRemoteSource(snapshot) ?? projectWorkspace?.cwd ?? null; } - catch { return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; } + try { + resolvedBaseCwd = recipe && isClosedExecutionWorkspaceStatus(snapshot.status) + ? await restoreGitCacheBase(snapshot, recipe, gitAuth) + : await verifiedRecordedIsolatedRemoteSource(snapshot) ?? projectWorkspace?.cwd ?? null; + } catch { + return { ok: false, code: "rebuild_failed", message: recipe + ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + } if (isClosedExecutionWorkspaceStatus(snapshot.status) && resolvedBaseCwd == null && snapshot.strategyType === "git_worktree" && snapshot.projectId) { try { const managedWorkspace = await ensureManagedProjectWorkspace({ @@ -3689,7 +3732,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); resolvedBaseCwd = managedWorkspace.cwd; } catch { - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } } let capturedSource: IsolatedWorkspaceSourceAccess | null = null; @@ -3709,7 +3752,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic actor: input.authorizationActor, captured: capturedSource, issueId: issue.id, phase: "preparing", }); } catch { - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } } const result = await db.transaction(async (tx): Promise => { @@ -3779,6 +3822,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const nextMetadata = bumpExecutionWorkspaceLifecycleGeneration( row.metadata as Record | null, ); + delete nextMetadata.gitCacheExpiration; const recorder = workspaceOperationService(db).createRecorder({ companyId: row.companyId, executionWorkspaceId: row.id, @@ -3815,6 +3859,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic if (copyOnlyProvisioning) nextMetadata[CONTROLLER_PROVISION_COPY_ONLY] = true; const realized = await ensurePersistedExecutionWorkspaceAvailable({ db: tx as unknown as Db, + resolveGitAuth: gitAuth, skipHostProvision: trust.kind === "low_trust_review" || copyOnlyProvisioning, beforeSourceCopy: verifyCopyIdentity, afterSourceCopy: verifyCopyIdentity, @@ -3909,7 +3954,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic generation: nextGeneration, }, }); - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } // The rebuild succeeded. Publish the row as active in one write, and clear @@ -3918,6 +3963,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic // the issue out of the terminal state only after this reopen returns. The // flag stops the terminal reaper and the archive route from archiving and // destroying the rebuilt worktree in that window. + // Keep the refresh guard after reopening so normal preparation cannot + // advance the saved checkout to upstream HEAD. Its intact branch may now + // receive newer task commits; the next expiration records a fresh recipe. + if (recipe) nextMetadata.gitCacheRestoration = { ...recipe, restoredAt: now().toISOString() }; const activeMetadata = setMetadataReopenPendingConsumption(nextMetadata, now()); await measureRunPhase("workspace.notification", () => notifyTerminalWorkspaceWork(tx, `workspace:${row.id}`)); const activeRow = await tx @@ -3935,7 +3984,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic .returning() .then((rows) => rows[0] ?? null); if (!activeRow) { - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } logger.info( { @@ -3967,7 +4016,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic return { ok: true, reopened: true, workspace: toExecutionWorkspace(activeRow), generation: nextGeneration }; }).catch((error): ReopenClosedIsolatedExecutionWorkspaceResult => { logger.warn({ error, executionWorkspaceId: snapshot.id }, "Execution workspace reopen failed"); - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; }); if (capturedSource && authorizedTarget && resolvedBaseCwd) { try { @@ -3981,10 +4030,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic workspaceId: snapshot.id, issue, actor, expectedGeneration: result.generation, }); } - return { ok: false, code: "rebuild_failed", message: "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } } return result; + }; + return authorizedSnapshot.sourceIssueId + ? withIsolatedRepositorySourceLock(db, authorizedSnapshot.companyId, authorizedSnapshot.sourceIssueId, reopen) + : reopen(); }, // Clear the reopen-pending flag after a caller failed to consume a reopened @@ -4373,6 +4426,13 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic branchName: inspection.toBranch, updatedAt: now, }; + const restoration = readGitCacheRestoration(lockedRow); + if (restoration) { + updatePatch.metadata = { + ...lockedRow.metadata, + gitCacheRestoration: { ...restoration, branchName: inspection.toBranch }, + }; + } if (lockedWorkspace.name === inspection.fromBranch) { updatePatch.name = inspection.toBranch; } diff --git a/server/src/services/git-workspace-expiration.ts b/server/src/services/git-workspace-expiration.ts new file mode 100644 index 00000000000..a6c39b670ae --- /dev/null +++ b/server/src/services/git-workspace-expiration.ts @@ -0,0 +1,230 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { and, eq, ne, sql } from "drizzle-orm"; +import { executionWorkspaces, issues, heartbeatRuns, nativeRunFinalizations, environmentLeases, + workspaceRuntimeServices, issueRecoveryActions, projectWorkspaces, withDedicatedDbConnection, type Db } from "@paperclipai/db"; +import { createGitRemoteAuthProvider } from "./git-credentials.js"; +import { withIsolatedRepositorySourceLock, verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; +import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, inspectGitCacheRecipe, readGitCacheRestoration, + verifyRemoteGitCommit, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js"; + + +async function allocatedBytes(target: string, shouldContinue: () => boolean): Promise { + if (!shouldContinue()) throw new Error("scan_stopped"); + const entry = await fs.lstat(target); + if (!entry.isDirectory() && !entry.isFile() && !entry.isSymbolicLink()) throw new Error("unsupported_filesystem_entry"); + let bytes = entry.blocks * 512; + if (entry.isDirectory()) for (const name of await fs.readdir(target)) bytes += await allocatedBytes(path.join(target, name), shouldContinue); + return bytes; +} +function skipReason(error: unknown) { + return error instanceof Error && /^[a-z][a-z_]+$/.test(error.message) ? error.message : "inspection_failed"; +} + +const EXPIRATION_KEY = "gitCacheExpiration"; +const DAY_MS = 24 * 60 * 60 * 1000; +type Workspace = typeof executionWorkspaces.$inferSelect; +type Tx = Parameters[0]>[0]; +type Expiration = GitCacheExpiration; +export type GitExpirationResult = { candidate: boolean; expired: boolean; reclaimedBytes: number; + candidateBytes: number; baseExpired: boolean; skipReason: string | null }; + +/** Re-read the whole tree and every runtime/recovery consumer immediately before deletion. */ +async function eligible(tx: Tx, workspace: Workspace, now: Date) { + if (workspace.metadata?.reopenPendingConsumption === true) return false; + const rows = await tx.execute<{ allowed: boolean }>(sql` + WITH RECURSIVE tree AS ( + SELECT id, status, completed_at, cancelled_at, updated_at FROM ${issues} + WHERE company_id = ${workspace.companyId} AND id = ${workspace.sourceIssueId} + UNION + SELECT c.id, c.status, c.completed_at, c.cancelled_at, c.updated_at FROM ${issues} c + JOIN tree p ON c.parent_id = p.id WHERE c.company_id = ${workspace.companyId} + ), relevant_runs AS ( + SELECT r.* FROM ${heartbeatRuns} r WHERE r.company_id = ${workspace.companyId} AND ( + r.issue_id IN (SELECT id FROM tree) OR r.native_issue_id IN (SELECT id FROM tree) + OR r.context_snapshot->>'issueId' IN (SELECT id::text FROM tree) + OR r.context_snapshot->>'taskId' IN (SELECT id::text FROM tree) + OR r.context_snapshot->>'executionWorkspaceId' = ${workspace.id} + OR r.id IN (SELECT unnest(ARRAY[checkout_run_id, execution_run_id]) FROM ${issues} + WHERE company_id = ${workspace.companyId} AND execution_workspace_id = ${workspace.id}) + ) + ) SELECT ( + EXISTS (SELECT 1 FROM tree) + AND NOT EXISTS (SELECT 1 FROM tree WHERE status NOT IN ('done', 'cancelled') + OR coalesce(CASE WHEN status = 'done' THEN completed_at ELSE cancelled_at END, updated_at) > ${new Date(now.getTime() - DAY_MS).toISOString()}::timestamptz) + AND NOT EXISTS (SELECT 1 FROM ${issues} WHERE company_id = ${workspace.companyId} + AND execution_workspace_id = ${workspace.id} AND status NOT IN ('done', 'cancelled')) + AND NOT EXISTS (SELECT 1 FROM relevant_runs WHERE status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled', 'interrupted') + OR finished_at IS NULL OR controller_lease_expires_at > now()) + AND NOT EXISTS (SELECT 1 FROM ${issueRecoveryActions} WHERE company_id = ${workspace.companyId} + AND source_issue_id IN (SELECT id FROM tree) AND status IN ('active', 'escalated')) + AND NOT EXISTS (SELECT 1 FROM ${nativeRunFinalizations} f WHERE f.company_id = ${workspace.companyId} + AND (f.issue_id IN (SELECT id FROM tree) OR f.run_id IN (SELECT id FROM relevant_runs)) + AND (f.phase <> 'committed' OR f.lease_owner IS NOT NULL OR f.recovery_state IS NOT NULL)) + AND NOT EXISTS (SELECT 1 FROM ${environmentLeases} l WHERE l.company_id = ${workspace.companyId} + AND (l.execution_workspace_id = ${workspace.id} OR l.heartbeat_run_id IN (SELECT id FROM relevant_runs)) + AND (l.status = 'pending_cleanup' OR l.cleanup_status = 'failed' OR l.metadata ? 'nativeWorkspaceExportResume' + OR (l.released_at IS NULL AND NOT coalesce(l.status = 'retained' AND l.lease_policy = 'reuse_by_environment' AND l.cleanup_status = 'success', false)))) + AND NOT EXISTS (SELECT 1 FROM ${workspaceRuntimeServices} WHERE company_id = ${workspace.companyId} + AND (execution_workspace_id = ${workspace.id} OR cwd = ${workspace.cwd} + OR left(cwd, length(${workspace.cwd}) + 1) = ${workspace.cwd} || '/') AND status <> 'stopped') + AND NOT EXISTS (SELECT 1 FROM ${executionWorkspaces} other WHERE other.id <> ${workspace.id} + AND (other.cwd = ${workspace.cwd} OR other.provider_ref = ${workspace.cwd} + OR left(other.cwd, length(${workspace.cwd}) + 1) = ${workspace.cwd} || '/')) + AND NOT EXISTS (SELECT 1 FROM ${projectWorkspaces} WHERE cwd = ${workspace.cwd} + OR left(cwd, length(${workspace.cwd}) + 1) = ${workspace.cwd} || '/') + ) AS allowed`); + return rows[0]?.allowed === true; +} + +export function hasPendingGitExpiration(workspace: Workspace) { + const state = workspace.metadata?.[EXPIRATION_KEY] as Expiration | undefined; + return state?.schema === "paperclip.git-cache-expiration.v1" && !state.completed; +} + +async function exists(target: string) { + return fs.lstat(target).then(() => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; }); +} + +/** Each ref/reflog object must be recoverable, and no other checkout may consume the base. */ +async function inspectBase(db: Db, workspace: Workspace, signal?: AbortSignal, allowCurrentWorktree = false) { + const recipe = readGitCacheRestoration(workspace)!; + if (recipe.sourceKind !== "isolated_remote") throw new Error("shared_base_preserved"); + const other = await db.select().from(executionWorkspaces).where(and(ne(executionWorkspaces.id, workspace.id), + sql`(${executionWorkspaces.metadata}->'isolatedRemoteSource'->>'companyId' = ${workspace.companyId} + AND ${executionWorkspaces.metadata}->'isolatedRemoteSource'->>'issueId' = ${workspace.sourceIssueId}) + OR ${executionWorkspaces.cwd} = ${recipe.sourcePath} OR ${executionWorkspaces.providerRef} = ${recipe.sourcePath}`)); + if (other.some(row => !["archived", "cleanup_failed"].includes(row.status) + || !readGitCacheRestoration(row) || !readGitCacheExpiration(row)?.workspaceRemoved)) throw new Error("base_has_consumers"); + const [projectConsumer] = await db.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.cwd, recipe.sourcePath)).limit(1); + if (projectConsumer) throw new Error("base_has_consumers"); + const [service] = await db.select({ id: workspaceRuntimeServices.id }).from(workspaceRuntimeServices).where(sql` + ${workspaceRuntimeServices.status} <> 'stopped' AND (${workspaceRuntimeServices.cwd} = ${recipe.sourcePath} + OR left(${workspaceRuntimeServices.cwd}, length(${recipe.sourcePath}) + 1) = ${recipe.sourcePath} || '/')`).limit(1); + if (service) throw new Error("base_has_live_services"); + await verifiedRecordedIsolatedRemoteSource(workspace); + const worktrees = (await gitCacheCommand(recipe.sourcePath, ["worktree", "list", "--porcelain"], signal)) + .split("\n").filter(line => line.startsWith("worktree ")).map(line => line.slice(9)); + const allowed = [await fs.realpath(recipe.sourcePath), ...(allowCurrentWorktree ? [await fs.realpath(recipe.worktreePath)] : [])]; + if (worktrees.some(cwd => !allowed.includes(cwd))) throw new Error("base_has_worktrees"); + await assertReproducibleGitFiles(recipe.sourcePath, signal); + await assertReproducibleBaseGitStorage(recipe.sourcePath, signal); + const refs = (await gitCacheCommand(recipe.sourcePath, ["for-each-ref", "--format=%(objectname) %(refname)"], signal)).trim().split("\n").filter(Boolean); + if (refs.some(ref => !/^[a-f0-9]{40} refs\/(heads|remotes\/origin)\//.test(ref))) throw new Error("unsupported_base_refs"); + const commits = new Set([...refs.map(ref => ref.split(" ")[0]!), + ...(await gitCacheCommand(recipe.sourcePath, ["reflog", "show", "--all", "--format=%H"], signal)).trim().split("\n").filter(Boolean)]); + if (commits.size > 50) throw new Error("base_ref_verification_budget"); + const auth = createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }); + for (const commit of commits) await verifyRemoteGitCommit(recipe.repoUrl, commit, auth, signal); + // Unreachable objects may contain unique rewritten work not present in a reflog. + const unreachable = await gitCacheCommand(recipe.sourcePath, ["fsck", "--no-reflogs", "--unreachable", "--no-progress"], signal); + if (unreachable.trim()) throw new Error("unreachable_base_objects"); + return allocatedBytes(recipe.sourcePath, () => { signal?.throwIfAborted(); return true; }); +} + +export async function expireReconstructableGitWorkspace(db: Db, snapshot: Workspace, options: { + dryRun: boolean; expireBase: boolean; now: Date; signal?: AbortSignal; + beforeDelete?: (workspace: Workspace) => Promise; +}): Promise { + const result: GitExpirationResult = { candidate: false, expired: false, reclaimedBytes: 0, + candidateBytes: 0, baseExpired: false, skipReason: null }; + if (!snapshot.sourceIssueId) return { ...result, skipReason: "missing_source_issue" }; + try { + return await withIsolatedRepositorySourceLock(db, snapshot.companyId, snapshot.sourceIssueId, () => + withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + const acquired = await tx.execute<{ locked: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${`execution_workspace_lifecycle:${snapshot.id}`}, 0)) as locked`); + if (!acquired[0]?.locked) throw new Error("lifecycle_busy"); + let workspace = (await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, snapshot.id), eq(executionWorkspaces.companyId, snapshot.companyId))))[0]; + if (!workspace || workspace.updatedAt.getTime() !== snapshot.updatedAt.getTime()) throw new Error("workspace_changed"); + if (!await eligible(tx, workspace, options.now)) throw new Error("live_or_unresolved_consumer"); + let state = ["active", "idle", "in_review"].includes(workspace.status) ? null : readGitCacheExpiration(workspace); + // A restored active workspace starts a new expiration cycle and records its new HEAD. + if (!hasPendingGitExpiration(workspace) && !["active", "idle", "in_review"].includes(workspace.status)) throw new Error("not_expirable"); + if (["active", "idle", "in_review"].includes(workspace.status)) state = null; + let recipe = state ? readGitCacheRestoration(workspace)! : await inspectGitCacheRecipe(workspace, + createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }), options.signal); + const present = await exists(recipe.worktreePath); + if (!state && !present) throw new Error("missing_unverified_worktree"); + result.candidate = true; + if (present) result.candidateBytes = await allocatedBytes(recipe.worktreePath, () => { options.signal?.throwIfAborted(); return true; }); + if (options.dryRun) { + if (options.expireBase) { + try { result.candidateBytes += await inspectBase(db, { ...workspace, + metadata: { ...workspace.metadata, [GIT_CACHE_RESTORATION_KEY]: recipe } }, options.signal, true); } + catch (error) { result.skipReason = skipReason(error); } + } + return result; + } + if (!state) { + const generation = Number(workspace.metadata?.lifecycleGeneration ?? 0) + 1; + const rawAdmin = (await gitCacheCommand(recipe.worktreePath, ["rev-parse", "--absolute-git-dir"], options.signal)).trim(); + const worktreeAdmin = path.join(recipe.sourcePath, ".git", "worktrees", path.basename(rawAdmin)); + const canonicalAdmin = path.join(await fs.realpath(recipe.sourcePath), ".git", "worktrees", path.basename(rawAdmin)); + if (await fs.realpath(worktreeAdmin) !== canonicalAdmin || await fs.realpath(rawAdmin) !== canonicalAdmin) { + throw new Error("unproven_worktree_admin"); + } + state = { schema: "paperclip.git-cache-expiration.v1", generation, workspaceRemoved: false, + worktreeTrash: path.join(path.dirname(recipe.worktreePath), `.expired-git-worktree-${workspace.id}-${generation}`), + worktreeAdmin, baseTrash: null, completed: false }; + if (await exists(state.worktreeTrash)) throw new Error("unknown_worktree_trash"); + // Commit the immutable restoration recipe BEFORE touching disk. The separate + // lifecycle-lock connection remains held through deletion and final state. + [workspace] = await db.update(executionWorkspaces).set({ status: "archived", closedAt: options.now, + cleanupEligibleAt: options.now, cleanupReason: GIT_CACHE_EXPIRATION_REASON, updatedAt: options.now, + metadata: { ...workspace.metadata, lifecycleGeneration: generation, [GIT_CACHE_RESTORATION_KEY]: recipe, [EXPIRATION_KEY]: state }, + }).where(eq(executionWorkspaces.id, workspace.id)).returning(); + } + if (!workspace) throw new Error("workspace_changed"); + const persist = async () => { + [workspace] = await db.update(executionWorkspaces).set({ metadata: { ...workspace!.metadata, [EXPIRATION_KEY]: state }, updatedAt: new Date() }) + .where(eq(executionWorkspaces.id, snapshot.id)).returning(); + }; + await options.beforeDelete?.(workspace); + if (!await eligible(tx, workspace, options.now)) throw new Error("live_or_unresolved_consumer"); + if (!state.workspaceRemoved && present) { + const { acquireGitWorktreeCleanupLock } = await import("./workspace-runtime.js"); + const lock = await acquireGitWorktreeCleanupLock(recipe.worktreePath); + try { + await assertReproducibleGitFiles(recipe.worktreePath, options.signal); + if ((await gitCacheCommand(recipe.worktreePath, ["rev-parse", "HEAD"], options.signal)).trim() !== recipe.commit) throw new Error("head_changed"); + await fs.rename(recipe.worktreePath, state.worktreeTrash); + await finishGitCacheTrash(workspace); + result.reclaimedBytes += result.candidateBytes; + } finally { await lock.release(); } + } + await finishGitCacheTrash(workspace); + state.workspaceRemoved = true; + await persist(); + result.expired = true; + if (options.expireBase) { + try { + if ((!state.baseTrash || !await exists(state.baseTrash)) && await exists(recipe.sourcePath)) { + const bytes = await inspectBase(db, workspace!, options.signal); + if (!await eligible(tx, workspace!, options.now)) throw new Error("live_or_unresolved_consumer"); + const trash = path.join(path.dirname(recipe.sourcePath), `.expired-git-base-${snapshot.id}-${state.generation}`); + if (await exists(trash)) throw new Error("unknown_base_trash"); + state.baseTrash = trash; + await persist(); + await fs.rename(recipe.sourcePath, trash); + result.candidateBytes += bytes; + } + if (state.baseTrash && await exists(state.baseTrash)) { + const stat = await fs.lstat(state.baseTrash); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("unsafe_base_trash"); + const bytes = await allocatedBytes(state.baseTrash, () => { options.signal?.throwIfAborted(); return true; }); + await fs.rm(state.baseTrash, { recursive: true, force: true }); + result.reclaimedBytes += bytes; + result.baseExpired = true; + } + } catch (error) { result.skipReason = skipReason(error); } + } + state.completed = recipe.sourceKind !== "isolated_remote" || !options.expireBase || (result.baseExpired || !await exists(recipe.sourcePath)) && (!state.baseTrash || !await exists(state.baseTrash)); + await persist(); + return result; + })), true); + } catch (error) { + options.signal?.throwIfAborted(); + result.skipReason = skipReason(error); + return result; + } +} diff --git a/server/src/services/git-workspace-restoration.ts b/server/src/services/git-workspace-restoration.ts new file mode 100644 index 00000000000..dd02daa52d1 --- /dev/null +++ b/server/src/services/git-workspace-restoration.ts @@ -0,0 +1,346 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execFile as execFileCallback } from "node:child_process"; +import { createHash } from "node:crypto"; +import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; +import { promisify } from "node:util"; +import type { GitRemoteAuthProvider } from "./git-credentials.js"; +import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js"; +import { recordedIsolatedRemoteSource, verifiedRecordedIsolatedRemoteSource, materializeIsolatedRemoteWorkspaceSource } from "./isolated-remote-workspace-source.js"; +import { findRegenerableWorkspaceDirs } from "./workspace-compaction.js"; + +const execFile = promisify(execFileCallback); +const shaPattern = /^[a-f0-9]{40}$/; +export const GIT_CACHE_RESTORATION_KEY = "gitCacheRestoration"; +export const GIT_CACHE_EXPIRATION_REASON = "reproducible_git_cache_expired"; +export type GitCacheRestoration = { + schema: "paperclip.git-cache-restoration.v1"; + companyId: string; + workspaceId: string; + sourceIssueId: string; + repoUrl: string; + commit: string; + branchName: string; + sourceKind: "isolated_remote" | "managed_project"; + projectId: string | null; + sourcePath: string; + worktreePath: string; + gitBranchOwnershipVersion: 1; + verifiedAt: string; + restoredAt?: string; +}; +export type GitCacheExpiration = { + schema: "paperclip.git-cache-expiration.v1"; generation: number; workspaceRemoved: boolean; + worktreeTrash: string; worktreeAdmin: string; baseTrash: string | null; completed: boolean; +}; +type Workspace = { + id: string; companyId: string; projectId: string | null; sourceIssueId: string | null; repoUrl: string | null; + mode: string | null; strategyType: string | null; branchName: string | null; + cwd: string | null; providerRef: string | null; metadata: Record | null; +}; + +function managedSourcePaths(workspace: Workspace) { + if (!workspace.projectId || !workspace.repoUrl) return []; + let repoName: string | null = null; + try { repoName = new URL(workspace.repoUrl).pathname.replace(/\/+$/, "").split("/").pop()?.replace(/\.git$/i, "") || null; } catch { return []; } + const root = resolveManagedProjectWorkspaceDir({ companyId: workspace.companyId, projectId: workspace.projectId, repoName }); + return [root, `${root}-${createHash("sha256").update(workspace.repoUrl).digest("hex").slice(0, 12)}`]; +} + +async function verifiedGitCacheSource(workspace: Workspace, sourcePath: string) { + const privateSource = recordedIsolatedRemoteSource(workspace); + if (privateSource) { + if (sourcePath !== privateSource) throw new GitCacheRestorationFailure(); + return (await verifiedRecordedIsolatedRemoteSource(workspace))!; + } + if (!managedSourcePaths(workspace).includes(sourcePath)) throw new GitCacheRestorationFailure(); + const root = resolvePaperclipInstanceRoot(); + const actual = await fs.realpath(sourcePath); + if (actual !== path.join(await fs.realpath(root), path.relative(root, sourcePath))) throw new GitCacheRestorationFailure(); + const storage = await fs.lstat(path.join(actual, ".git")); + if (!storage.isDirectory() || storage.isSymbolicLink() + || (await gitCacheCommand(actual, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim() !== path.join(actual, ".git") + || (await gitCacheCommand(actual, ["config", "--local", "--null", "--get-all", "remote.origin.url"])) !== `${workspace.repoUrl}\0`) throw new GitCacheRestorationFailure(); + return sourcePath; +} + +export class GitCacheRestorationFailure extends Error { + constructor() { super("The saved workspace commit could not be restored. Check repository access and retry."); } +} + +export function readGitCacheRestoration(workspace: Workspace): GitCacheRestoration | null { + const value = workspace.metadata?.[GIT_CACHE_RESTORATION_KEY]; + if (value === undefined) return null; + const recipe = value as GitCacheRestoration | null; + if (!recipe || recipe.schema !== "paperclip.git-cache-restoration.v1" + || !isRuntimeOwnedGitBranch(workspace.metadata) + || recipe.gitBranchOwnershipVersion !== 1 + || recipe.companyId !== workspace.companyId || recipe.workspaceId !== workspace.id + || recipe.sourceIssueId !== workspace.sourceIssueId || recipe.repoUrl !== workspace.repoUrl + || recipe.branchName !== workspace.branchName || recipe.worktreePath !== workspace.cwd + || (workspace.providerRef && recipe.worktreePath !== workspace.providerRef) + || recipe.projectId !== workspace.projectId + || (recipe.sourceKind === "isolated_remote" ? recipe.sourcePath !== recordedIsolatedRemoteSource(workspace) + : recipe.sourceKind !== "managed_project" || recordedIsolatedRemoteSource(workspace) !== null || !managedSourcePaths(workspace).includes(recipe.sourcePath)) + || !shaPattern.test(recipe.commit) || !Number.isFinite(Date.parse(recipe.verifiedAt)) + || (recipe.restoredAt !== undefined && !Number.isFinite(Date.parse(recipe.restoredAt)))) { + throw new GitCacheRestorationFailure(); + } + return recipe; +} + +export async function gitCacheCommand(cwd: string, args: string[], signal?: AbortSignal) { + const result = await execFile("git", ["-C", cwd, ...args], { + timeout: 60_000, signal, maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" }, + }); + return result.stdout; +} + +/** A fresh object store proves the object came from upstream, not our cache. */ +export async function verifyRemoteGitCommit(repoUrl: string, commit: string, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal) { + if (!shaPattern.test(commit)) throw new GitCacheRestorationFailure(); + const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-proof-")); + try { + const auth = await resolveGitAuth(repoUrl); + const env = { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull }; + const options = { timeout: 60_000, signal, maxBuffer: 1024 * 1024, env }; + await execFile("git", ["init", "--bare", temporary], options); + await execFile("git", [...auth?.configArgs ?? [], "-C", temporary, "fetch", "--no-tags", "--no-recurse-submodules", "--depth=1", "--", repoUrl, commit], options); + const result = await execFile("git", ["-C", temporary, "rev-parse", "FETCH_HEAD^{commit}"], options); + if (result.stdout.trim() !== commit) throw new GitCacheRestorationFailure(); + } catch { throw new GitCacheRestorationFailure(); } + finally { await fs.rm(temporary, { recursive: true, force: true }); } +} + +let templateFiles: Promise> | undefined; +async function stockGitFiles() { + templateFiles ??= (async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-template-")); + const files = new Map(); + try { + await execFile("git", ["init", "--bare", root], { timeout: 10_000, + env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull } }); + for (const name of ["description", "info/exclude", ...(await fs.readdir(path.join(root, "hooks"))).map(name => `hooks/${name}`)]) { + files.set(name, await fs.readFile(path.join(root, name))); + } + return files; + } finally { await fs.rm(root, { recursive: true, force: true }); } + })(); + return templateFiles; +} + +export async function assertReproducibleBaseGitStorage(cwd: string, signal?: AbortSignal) { + const root = path.join(cwd, ".git"); + const allowed = new Set(["HEAD", "ORIG_HEAD", "FETCH_HEAD", "index", "config", "description", "hooks", "info", "refs", "objects", "logs", "packed-refs", "worktrees"]); + if ((await fs.readdir(root)).some(name => !allowed.has(name))) throw new Error("unknown_base_git_files"); + const templates = await stockGitFiles(); + for (const directory of ["hooks", "info"]) { + for (const name of await fs.readdir(path.join(root, directory))) { + const file = `${directory}/${name}`; + const entry = await fs.lstat(path.join(root, file)); + if (!entry.isFile() || !templates.get(file)?.equals(await fs.readFile(path.join(root, file)))) throw new Error("custom_git_metadata"); + } + } + if (!templates.get("description")?.equals(await fs.readFile(path.join(root, "description")))) throw new Error("custom_git_metadata"); + // A custom hook or metadata file inside object/ref storage must not disappear + // merely because Git's object/ref enumeration does not mention it. + async function walk(relative: string): Promise { + signal?.throwIfAborted(); + for (const entry of await fs.readdir(path.join(root, relative), { withFileTypes: true })) { + const name = `${relative}/${entry.name}`; + if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error("unsupported_git_storage"); + if (entry.isDirectory()) { await walk(name); continue; } + const valid = /^objects\/[0-9a-f]{2}\/[0-9a-f]{38}$/.test(name) + || /^objects\/pack\/pack-[0-9a-f]{40}\.(pack|idx|rev|bitmap)$/.test(name) + || /^(refs|logs\/refs)\/(heads|remotes\/origin)\//.test(name) + || name === "logs/HEAD"; + if (!valid) throw new Error("unknown_git_storage_file"); + } + } + for (const directory of ["objects", "refs", "logs"]) await walk(directory); +} + +/** Inspect every untracked/ignored file. Only repository-ignored regenerable directories qualify. */ +export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal) { + if ((await gitCacheCommand(cwd, ["status", "--porcelain=v1", "--untracked-files=all", "-z"], signal)).length) throw new Error("dirty_or_untracked_files"); + if ((await gitCacheCommand(cwd, ["ls-files", "-v"], signal)).split("\n").some(line => /^[a-zS] /.test(line))) throw new Error("unsupported_index_flags"); + const localConfig = await gitCacheCommand(cwd, ["config", "--local", "--name-only", "--list"], signal); + if (localConfig.split("\n").filter(Boolean).some(key => !/^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\.[^.]+\.(remote|merge))$/.test(key))) throw new Error("unsupported_local_git_config"); + const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal }); + const ignored = (await gitCacheCommand(cwd, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], signal)).split("\0").filter(Boolean); + if (ignored.some(file => !regenerable.some(dir => file.startsWith(`${dir}/`)))) throw new Error("unique_ignored_files"); + const entries = await gitCacheCommand(cwd, ["ls-tree", "-r", "HEAD"], signal); + if (/^160000 /m.test(entries)) throw new Error("unsupported_submodules"); + const gitDir = (await gitCacheCommand(cwd, ["rev-parse", "--absolute-git-dir"], signal)).trim(); + const common = (await gitCacheCommand(cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim(); + for (const root of new Set([gitDir, common])) { + for (const name of ["shallow", "shallow.lock", "MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "BISECT_LOG", "rebase-apply", "rebase-merge", "sequencer", "info/sparse-checkout", "objects/info/alternates", "refs/replace", "refs/stash", "modules", "lfs"]) { + if (await fs.lstat(path.join(root, name)).then(() => true, (e: NodeJS.ErrnoException) => { if(e.code === "ENOENT") return false; throw e; })) throw new Error("unsupported_local_git_state"); + } + } + const original = await gitCacheCommand(cwd, ["rev-parse", "--verify", "ORIG_HEAD"], signal).catch(() => null); + if (original) await gitCacheCommand(cwd, ["merge-base", "--is-ancestor", original.trim(), "HEAD"], signal) + .catch(() => { throw new Error("unique_original_head"); }); + if (gitDir !== common) { + const allowed = new Set(["HEAD", "ORIG_HEAD", "index", "commondir", "gitdir", "logs", "refs", "COMMIT_EDITMSG", "index.lock", "HEAD.lock"]); + if ((await fs.readdir(gitDir)).some(name => !allowed.has(name))) throw new Error("unknown_worktree_git_files"); + async function assertEmptyRefs(root: string): Promise { + const entries = await fs.readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; throw error; + }); + for (const entry of entries) { + if (!entry.isDirectory()) throw new Error("unique_worktree_refs"); + await assertEmptyRefs(path.join(root, entry.name)); + } + } + await assertEmptyRefs(path.join(gitDir, "refs")); + const message = await fs.readFile(path.join(gitDir, "COMMIT_EDITMSG"), "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; throw error; + }); + if (message !== null && message.trim() !== (await gitCacheCommand(cwd, ["log", "-1", "--format=%B"], signal)).trim()) throw new Error("uncommitted_git_message"); + const logs = await fs.readdir(path.join(gitDir, "logs")).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; throw error; + }); + if (logs.some(name => name !== "HEAD")) throw new Error("unknown_worktree_git_logs"); + } + // Packed stashes/replacement refs are just as important as loose refs. + if ((await gitCacheCommand(cwd, ["for-each-ref", "--format=%(refname)", "refs/stash", "refs/replace"], signal)).trim()) throw new Error("unique_git_refs"); + // Rewrites can leave commits reachable only from this worktree's reflog. + for (const commit of new Set((await gitCacheCommand(cwd, ["reflog", "show", "--format=%H", "HEAD"], signal)).split("\n").filter(Boolean))) { + await gitCacheCommand(cwd, ["merge-base", "--is-ancestor", commit, "HEAD"], signal).catch(() => { throw new Error("unique_reflog_commits"); }); + } + return regenerable; +} + +export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal): Promise { + if (workspace.mode !== "isolated_workspace" || workspace.strategyType !== "git_worktree" + || !isRuntimeOwnedGitBranch(workspace.metadata) || !workspace.sourceIssueId || !workspace.repoUrl + || !workspace.cwd || !workspace.branchName || (workspace.providerRef && workspace.providerRef !== workspace.cwd)) throw new Error("unproven_workspace_ownership"); + let source = await verifiedRecordedIsolatedRemoteSource(workspace); + if (!source) { + const common = (await gitCacheCommand(workspace.cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim(); + for (const candidate of managedSourcePaths(workspace)) { + if (await fs.realpath(path.join(candidate, ".git")).catch(() => null) === common) { + source = await verifiedGitCacheSource(workspace, candidate); break; + } + } + } + if (!source) throw new Error("unproven_source_ownership"); + const stat = await fs.lstat(workspace.cwd); + if (!stat.isDirectory() || stat.isSymbolicLink() || await fs.realpath(workspace.cwd) !== path.join(await fs.realpath(path.dirname(workspace.cwd)), path.basename(workspace.cwd))) throw new Error("unsafe_worktree_path"); + if (!(await fs.lstat(path.join(workspace.cwd, ".git"))).isFile()) throw new Error("not_linked_worktree"); + if ((await gitCacheCommand(workspace.cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim() !== path.join(await fs.realpath(source), ".git")) throw new Error("source_identity_changed"); + if ((await gitCacheCommand(workspace.cwd, ["symbolic-ref", "--short", "HEAD"], signal)).trim() !== workspace.branchName) throw new Error("branch_identity_changed"); + await assertReproducibleGitFiles(workspace.cwd, signal); + const commit = (await gitCacheCommand(workspace.cwd, ["rev-parse", "HEAD"], signal)).trim(); + await verifyRemoteGitCommit(workspace.repoUrl, commit, resolveGitAuth, signal); + return { schema: "paperclip.git-cache-restoration.v1", companyId: workspace.companyId, workspaceId: workspace.id, + sourceIssueId: workspace.sourceIssueId, repoUrl: workspace.repoUrl, commit, branchName: workspace.branchName, + sourceKind: recordedIsolatedRemoteSource(workspace) ? "isolated_remote" : "managed_project", projectId: workspace.projectId, + sourcePath: source, worktreePath: workspace.cwd, gitBranchOwnershipVersion: 1, verifiedAt: new Date().toISOString() }; +} + +export function readGitCacheExpiration(workspace: Workspace): GitCacheExpiration | null { + const state = workspace.metadata?.gitCacheExpiration as GitCacheExpiration | undefined; + if (!state) return null; + const recipe = readGitCacheRestoration(workspace); + if (!recipe || state.schema !== "paperclip.git-cache-expiration.v1" || !Number.isSafeInteger(state.generation) + || state.generation !== workspace.metadata?.lifecycleGeneration + || state.worktreeTrash !== path.join(path.dirname(recipe.worktreePath), `.expired-git-worktree-${workspace.id}-${state.generation}`) + || path.dirname(state.worktreeAdmin) !== path.join(recipe.sourcePath, ".git", "worktrees") + || (state.baseTrash !== null && state.baseTrash !== path.join(path.dirname(recipe.sourcePath), `.expired-git-base-${workspace.id}-${state.generation}`))) { + throw new GitCacheRestorationFailure(); + } + return state; +} + +// Both restart recovery and reopening finish only recorded, uniquely named trash. +// No directory scan or age heuristic can authorize deleting an arbitrary path. +export async function finishGitCacheTrash(workspace: Workspace) { + const state = readGitCacheExpiration(workspace); + if (!state) return; + const recipe = readGitCacheRestoration(workspace)!; + const exists = (target: string) => fs.lstat(target).then(() => true, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; throw error; + }); + for (const target of [state.worktreeTrash, state.baseTrash]) { + if (!target || !await exists(target)) continue; + const entry = await fs.lstat(target); + if (!entry.isDirectory() || entry.isSymbolicLink() + || await fs.realpath(target) !== path.join(await fs.realpath(path.dirname(target)), path.basename(target))) throw new GitCacheRestorationFailure(); + await fs.rm(target, { recursive: true, force: true }); + } + if (!await exists(recipe.worktreePath) && await exists(state.worktreeAdmin)) { + await verifiedGitCacheSource(workspace, recipe.sourcePath); + const entry = await fs.lstat(state.worktreeAdmin); + if (!entry.isDirectory() || entry.isSymbolicLink() + || await fs.realpath(state.worktreeAdmin) !== path.join(await fs.realpath(recipe.sourcePath), ".git", "worktrees", path.basename(state.worktreeAdmin))) { + throw new GitCacheRestorationFailure(); + } + const pointer = (await fs.readFile(path.join(state.worktreeAdmin, "gitdir"), "utf8")).trim(); + const expected = path.join(await fs.realpath(path.dirname(recipe.worktreePath)), path.basename(recipe.worktreePath), ".git"); + if (pointer !== expected) throw new GitCacheRestorationFailure(); + await fs.rm(state.worktreeAdmin, { recursive: true, force: true }); + } +} + +/** Called only after authorization and while holding the source/lifecycle fences. */ +export async function restoreGitCacheBase(workspace: Workspace, recipe: GitCacheRestoration, resolveGitAuth: GitRemoteAuthProvider) { + if (readGitCacheRestoration(workspace)?.commit !== recipe.commit) throw new GitCacheRestorationFailure(); + try { + await finishGitCacheTrash(workspace); + const present = await fs.lstat(recipe.sourcePath).then(() => true, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; + throw error; + }); + if (!present && recipe.sourceKind === "managed_project") { + const root = resolvePaperclipInstanceRoot(); + let parent = await fs.realpath(root); + for (const segment of path.relative(root, path.dirname(recipe.sourcePath)).split(path.sep)) { + parent = path.join(parent, segment); + await fs.mkdir(parent, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; }); + const entry = await fs.lstat(parent); + if (!entry.isDirectory() || entry.isSymbolicLink() || await fs.realpath(parent) !== parent) throw new GitCacheRestorationFailure(); + } + } + const cwd = present ? await verifiedGitCacheSource(workspace, recipe.sourcePath) + : recipe.sourceKind === "isolated_remote" + ? await materializeIsolatedRemoteWorkspaceSource({ companyId: recipe.companyId, issueId: recipe.sourceIssueId, + repoUrl: recipe.repoUrl, resolveGitAuth }) + : (await materializeManagedProjectWorkspace(recipe.sourcePath, { repoUrl: recipe.repoUrl, resolveGitAuth })).cwd; + await verifiedGitCacheSource(workspace, recipe.sourcePath); + if (!cwd) throw new GitCacheRestorationFailure(); + await ensureSavedGitCommit(cwd, recipe.repoUrl, recipe.commit, resolveGitAuth); + return cwd; + } catch { throw new GitCacheRestorationFailure(); } +} + +export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit: string, resolveGitAuth: (url: string) => Promise<{ configArgs: string[]; env: Record } | null>) { + if (!shaPattern.test(commit)) throw new GitCacheRestorationFailure(); + if (await gitCacheCommand(cwd, ["cat-file", "-e", `${commit}^{commit}`]).then(() => true, () => false)) return; + try { + const auth = await resolveGitAuth(repoUrl); + await execFile("git", [...auth?.configArgs ?? [], "-C", cwd, "fetch", "--no-tags", "--no-recurse-submodules", "--", repoUrl, commit], { + timeout: 60_000, maxBuffer: 1024 * 1024, env: { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0" }, + }); + if ((await gitCacheCommand(cwd, ["rev-parse", `${commit}^{commit}`])).trim() !== commit) throw new GitCacheRestorationFailure(); + } catch { throw new GitCacheRestorationFailure(); } +} + +/** Runtime callers must bind the recipe to the selected workspace and base. */ +export function savedGitWorkspaceRestoration(input: { metadata?: Record | null; companyId: string; + workspaceId?: string | null; cwd: string; baseCwd: string; repoUrl: string | null; branchName: string | null }) { + const raw = input.metadata?.[GIT_CACHE_RESTORATION_KEY]; + if (raw === undefined) return null; + const value = raw as GitCacheRestoration | null; + if (!value || value.schema !== "paperclip.git-cache-restoration.v1" || !shaPattern.test(value.commit) + || value.companyId !== input.companyId || value.workspaceId !== input.workspaceId + || value.worktreePath !== input.cwd || value.sourcePath !== input.baseCwd + || value.repoUrl !== input.repoUrl || value.branchName !== input.branchName + || !isRuntimeOwnedGitBranch(input.metadata) + || (value.restoredAt !== undefined && !Number.isFinite(Date.parse(value.restoredAt)))) throw new GitCacheRestorationFailure(); + return { commit: value.commit, restored: value.restoredAt !== undefined }; +} diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 92cb041602a..991398f1d4c 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -30,7 +30,7 @@ import { readNativeWorkspaceSyncReference, readNativeWorkspaceSyncLocalCwd } fro import { parseObject } from "../../adapters/utils.js"; import { materializeNativeChatTaskRoot } from "../native-runtime/native-chat-workspace.js"; import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js"; -import { canIsolateRemoteWorkspaceSource, materializeIsolatedRemoteWorkspaceSource } from "../isolated-remote-workspace-source.js"; +import { canIsolateRemoteWorkspaceSource, materializeIsolatedRemoteWorkspaceSource, withIsolatedRepositorySourceLock } from "../isolated-remote-workspace-source.js"; import { assertIsolatedWorkspaceSourceReadAccess, retainIsolatedWorkspaceSourceAccess, @@ -149,6 +149,14 @@ function readNonEmptyString(value: unknown): string | null { } export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) { + const sourceIssueId = input.reuse.reusableExistingExecutionWorkspace?.sourceIssueId + ?? input.reuse.existingExecutionWorkspace?.sourceIssueId ?? input.task.issueId; + if (!sourceIssueId || input.reuse.isDotRun) return prepareHeartbeatWorkspaceLocked(db, input); + return withIsolatedRepositorySourceLock(db, input.agent.companyId, sourceIssueId, + () => prepareHeartbeatWorkspaceLocked(db, input)); +} + +async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspacePreparationInput) { const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input; const timeWorkspace = createHeartbeatPreparationTiming(run.id); const { diff --git a/server/src/services/isolated-remote-workspace-source.ts b/server/src/services/isolated-remote-workspace-source.ts index efb299db2e1..02357b3ba35 100644 --- a/server/src/services/isolated-remote-workspace-source.ts +++ b/server/src/services/isolated-remote-workspace-source.ts @@ -1,3 +1,6 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { withDedicatedDbConnection, type Db } from "@paperclipai/db"; +import { sql } from "drizzle-orm"; import path from "node:path"; import fs from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; @@ -10,6 +13,25 @@ import { measureRunPhase } from "./execution-preparation-timing.js"; const execFile = promisify(execFileCallback); +const sourceLocks = new AsyncLocalStorage>(); +export class IsolatedRepositorySourceBusy extends Error {} + +/** Creation, reopening and base expiration share this cross-process exclusion. */ +export async function withIsolatedRepositorySourceLock(db: Db, companyId: string, issueId: string, + action: () => Promise, tryOnly = false): Promise { + const key = `isolated_repository_source:${companyId}:${issueId}`; + const held = sourceLocks.getStore() ?? []; + if (held.some(lock => lock.db === db && lock.key === key)) return action(); + return withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + if (tryOnly) { + const rows = await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${key}, 0)) as acquired`); + if (!rows[0]?.acquired) throw new IsolatedRepositorySourceBusy(); + } else await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${key}, 0))`); + return sourceLocks.run([...held, { db, key }], action); + })); +} + + export function isolatedRemoteSourcePath(companyId: string, issueId: string): string { if ([companyId, issueId].some(id => !/^[a-zA-Z0-9_-]+$/.test(id))) { throw new Error("Invalid isolated repository source identity"); diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index d497fd1feeb..97b393f8424 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -1,3 +1,4 @@ +import { savedGitWorkspaceRestoration, ensureSavedGitCommit, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; import { verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; export { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; @@ -3848,6 +3849,10 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { "Cannot rebuild the git worktree: the base project checkout directory does not exist.", ); } + const restoration = savedGitWorkspaceRestoration({ metadata: input.workspace.metadata, companyId: input.agent.companyId, + workspaceId: input.workspace.id, cwd, baseCwd, repoUrl: input.workspace.repoUrl ?? input.base.repoUrl, + branchName: input.workspace.branchName ?? null }); + const savedCommit = restoration?.commit ?? null; const repoRoot = await runGit(["rev-parse", "--show-toplevel"], baseCwd); await input.beforeSourceCopy?.(repoRoot, realized.worktreePath ?? cwd); const recordedBaseRefSha = readRecordedBaseRefSha(input.workspace.metadata); @@ -3908,7 +3913,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { const currentBaseRefSha = reuseBaseRef ? await resolveBaseRefSha(repoRoot, reuseBaseRef) : null; // An unstarted-worktree refresh can fast-forward the checked-out branch. // Never run it for an attached operator-owned ref. - const refresh = realized.branchCreatedByRuntime && reuseBaseRef && currentBaseRefSha + const refresh = !savedCommit && realized.branchCreatedByRuntime && reuseBaseRef && currentBaseRefSha ? await refreshUnstartedWorktreeToBase({ repoRoot, worktreePath: reuseWorktreePath, @@ -3955,7 +3960,14 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { await fs.mkdir(path.dirname(worktreePath), { recursive: true }); await runGit(["worktree", "prune"], repoRoot).catch(() => {}); - const restoreBaseRef = input.workspace.baseRef ?? input.base.repoRef ?? null; + if (savedCommit) { + await ensureSavedGitCommit(repoRoot, input.workspace.repoUrl ?? input.base.repoUrl!, savedCommit, input.resolveGitAuth ?? (async () => null)); + const existingTip = await resolveBaseRefSha(repoRoot, `refs/heads/${branchName}`); + // Once restored, subsequent task commits own this branch. The saved commit + // still prevents a clean resumed worktree from refreshing to upstream HEAD. + if (existingTip && existingTip !== savedCommit && !restoration?.restored) throw new GitCacheRestorationFailure(); + } + const restoreBaseRef = savedCommit ? null : input.workspace.baseRef ?? input.base.repoRef ?? null; const restoreRefreshWarnings = restoreBaseRef ? await refreshRemoteTrackingBaseRef(repoRoot, restoreBaseRef, input.resolveGitAuth) : []; @@ -3992,7 +4004,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { `Execution workspace "${worktreePath}" cannot be restored because its operator-owned branch "${branchName}" no longer exists.`, ); } - const baseRef = input.workspace.baseRef ?? await detectDefaultBranch(repoRoot) ?? "HEAD"; + const baseRef = savedCommit ?? input.workspace.baseRef ?? await detectDefaultBranch(repoRoot) ?? "HEAD"; const recreatedBaseRefSha = await resolveBaseRefSha(repoRoot, baseRef); await recordGitOperation(input.recorder, { phase: "worktree_prepare",