From 131b566ca4fa866c4348739c75cfe48c805b3398 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 6 Nov 2026 21:33:59 -0800 Subject: [PATCH] fix(workspaces): report safe Git restore diagnostics (#19736) Return the restore stage and failure category with the existing HTTP409 rejection. Preserve the unsaved comment and editable draft. Keep nested saved-commit and clone-timeout diagnostics through preparation wrappers, and distinguish local Git file errors from authentication failures. Retain only safe process flags privately; no raw command cause is exposed. All seven affected local suites pass (288 tests) and the layered server typecheck passes on supported Node25.6.1. Reviewed head 022eb7c2eb53a3d2e498fb223cea218efe667c3a has Greptile Apex5/5, zero unresolved findings and all applicable CI passing (55 successful checks and4 intentional skips). Co-Authored-By: Sentry Bot Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Paperclip --- doc/DEVELOPING.md | 10 +++ .../git-workspace-expiration.test.ts | 72 +++++++++++++++++- ...t-workspace-restoration-diagnostic.test.ts | 59 +++++++++++++++ .../issue-closed-workspace-routes.test.ts | 3 +- .../managed-repository-checkout.test.ts | 25 ++++++- server/src/routes/issues.ts | 2 +- server/src/services/execution-workspaces.ts | 52 +++++++++---- server/src/services/git-workspace-contents.ts | 48 ++++++------ .../src/services/git-workspace-restoration.ts | 73 +++++++++++++++++-- .../services/managed-repository-checkout.ts | 28 ++++++- 10 files changed, 325 insertions(+), 47 deletions(-) create mode 100644 server/src/__tests__/git-workspace-restoration-diagnostic.test.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index a4c931f945a..aa0ecf30a0a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -4533,6 +4533,16 @@ retryable restoration failure. It never substitutes the default branch. Reopening failures reject the consuming request with HTTP 409 before saving a comment or changing its task. The response preserves the restoration message, so the composer keeps an editable draft and permits retry after access is restored. +It also carries `diagnostic: { step, kind }`, and the server log records the same +pair. Both are closed values: `step` names the failed stage (for example +`source_materialize`, `restore_point` or `saved_commit`), and `kind` names the +failure class (`remote_unavailable`, `auth_rejected`, `commit_unavailable`, +`timeout`, `verification`, `filesystem`, `git_command` or `unknown`). They never +contain paths, repository URLs or Git output. +Nested checkout failures retain their original restore step. The checkout +wrapper keeps process error codes and timeout flags privately for classification; +it does not attach the raw command error as a cause or add those flags to the +public error object. Both enable switches default off. Rollback disables further expiration while leaving saved recipes and restoration support intact. Recovery archives, runner diff --git a/server/src/__tests__/git-workspace-expiration.test.ts b/server/src/__tests__/git-workspace-expiration.test.ts index ee42fda684d..53a4c36776e 100644 --- a/server/src/__tests__/git-workspace-expiration.test.ts +++ b/server/src/__tests__/git-workspace-expiration.test.ts @@ -28,6 +28,21 @@ import * as activityLog from "../services/activity-log.js"; import { readGitCacheRestoration } from "../services/git-workspace-restoration.js"; import { assertReproducibleManagedCheckout, gitCacheExpirationLockLocations, readGitCacheExpiration } from "../services/git-workspace-restoration.js"; +const cloneFailure = vi.hoisted(() => ({ error: null as Error | null })); +vi.mock("node:child_process", async importOriginal => { + const actual = await importOriginal(); + const { promisify } = await import("node:util"); + const execute = promisify(actual.execFile); + const execFile = (...args: unknown[]) => Reflect.apply(actual.execFile, undefined, args); + Object.defineProperty(execFile, promisify.custom, { value: (...args: unknown[]) => { + if (cloneFailure.error && args[0] === "git" && Array.isArray(args[1]) && args[1].includes("clone")) { + return Promise.reject(cloneFailure.error); + } + return Reflect.apply(execute, undefined, args); + } }); + return { ...actual, execFile }; +}); + const gitEnv = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test", GIT_TERMINAL_PROMPT: "0" }; const git = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, env: gitEnv, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); @@ -986,13 +1001,68 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { const f = await fixture(); expect((await f.expire()).baseExpired).toBe(true); await fs.rename(f.remote, `${f.remote}.offline`); - expect(await f.reopen()).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry." }); + const failed = await f.reopen(); + expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry.", + diagnostic: { step: "source_materialize", kind: "remote_unavailable" } }); + expect(JSON.stringify(failed)).not.toContain(root); expect(await f.reopen(randomUUID())).toMatchObject({ ok: false, code: "not_reopenable" }); expect(await exists(f.cwd)).toBe(false); await fs.rename(`${f.remote}.offline`, f.remote); expect(await f.reopen()).toMatchObject({ ok: true }); expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); }); + it("reports the failed step and kind when the saved commit is gone from upstream", async () => { + const f = await fixture(); + expect((await f.expire()).baseExpired).toBe(true); + git(f.source, "checkout", "--orphan", "rewritten"); + await fs.writeFile(path.join(f.source, "marker.txt"), "rewritten\n"); + git(f.source, "add", "."); git(f.source, "commit", "-m", "Rewritten"); + git(f.source, "push", "--force", "origin", "rewritten:main"); + git(f.remote, "reflog", "expire", "--expire=now", "--all"); git(f.remote, "gc", "--prune=now"); + const failed = await f.reopen(); + expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry.", + diagnostic: { step: "saved_commit", kind: "commit_unavailable" } }); + expect(JSON.stringify(failed)).not.toContain(root); + expect(JSON.stringify(failed)).not.toContain(f.commitA); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA }); + }); + + it("retains the nested saved commit failure through checkout preparation and reopening", async () => { + const f = await nestedFixture(true, false); + expect((await f.expire({ expireBase: false })).expired).toBe(true); + git(f.nestedSource, "checkout", "--orphan", "rewritten"); + await fs.writeFile(path.join(f.nestedSource, "nested.txt"), "rewritten\n"); + git(f.nestedSource, "add", "."); git(f.nestedSource, "commit", "-m", "Rewritten nested history"); + git(f.nestedSource, "push", "--force", f.nestedRemote, "rewritten:main"); + git(f.nestedRemote, "reflog", "expire", "--expire=now", "--all"); + git(f.nestedRemote, "gc", "--prune=now"); + const failed = await f.reopen(); + expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", + diagnostic: { step: "saved_commit", kind: "commit_unavailable" } }); + expect(JSON.stringify(failed)).not.toContain(root); + expect(JSON.stringify(failed)).not.toContain(f.nestedA); + expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ contents: { + checkouts: [expect.objectContaining({ commit: f.nestedA })], + } }); + }); + + it("retains a managed clone timeout through preparation and reopening", async () => { + const f = await fixture(true, false); + expect((await f.expire({ expireBase: false })).expired).toBe(true); + await fs.rename(f.baseCwd, `${f.baseCwd}.offline`); + cloneFailure.error = Object.assign(new Error("clone timed out; fixture credential must stay private"), { + killed: true, signal: "SIGTERM", code: null, + }); + let failed: Awaited>; + try { failed = await f.reopen(); } + finally { cloneFailure.error = null; } + expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", + diagnostic: { step: "source_materialize", kind: "timeout" } }); + expect(JSON.stringify(failed)).not.toContain(root); + expect(JSON.stringify(failed)).not.toContain("fixture credential"); + expect(await f.reopen()).toMatchObject({ ok: true }); + }); + it("serializes concurrent reopening after deletion and publishes one transition", async () => { const f = await fixture(); let release!: () => void, entered!: () => void; diff --git a/server/src/__tests__/git-workspace-restoration-diagnostic.test.ts b/server/src/__tests__/git-workspace-restoration-diagnostic.test.ts new file mode 100644 index 00000000000..bc92845cc87 --- /dev/null +++ b/server/src/__tests__/git-workspace-restoration-diagnostic.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { classifyWorkspaceRestorationError, GitCacheRestorationFailure } from "../services/git-workspace-restoration.js"; +import { workspaceReopenDiagnostic } from "../services/execution-workspaces.js"; +import { GitConnectionFailureError } from "../services/git-connection-failure.js"; + +const gitError = (stderr: string, extra: Record = {}) => + Object.assign(new Error(`Command failed: git fetch\n${stderr}`), { code: 128, stderr, ...extra }); + +describe("workspace restoration diagnostic", () => { + it.each([ + ["fatal: Authentication failed for 'https://user:sk-secret@example.test/repo.git/'", "auth_rejected"], + ["fatal: could not read Username for 'https://github.com': terminal prompts disabled", "auth_rejected"], + ["fatal: remote error: upload-pack: not our ref 9a204a56", "commit_unavailable"], + ["fatal: couldn't find remote ref refs/heads/gone", "commit_unavailable"], + ["fatal: unable to access 'https://example.test/repo.git/': Could not resolve host: example.test", "remote_unavailable"], + ["fatal: '/srv/repo.git' does not appear to be a git repository", "remote_unavailable"], + ["fatal: bad revision", "git_command"], + ["error: cannot open '.git/FETCH_HEAD': Permission denied", "filesystem"], + ["fatal: unable to access '/private/repository/.git/config': Permission denied", "filesystem"], + ["git@example.test: Permission denied (publickey).", "auth_rejected"], + ["git@example.test: Permission denied (publickey).\nerror: cannot open '.git/FETCH_HEAD': Permission denied", "filesystem"], + ])("classifies %s", (stderr, kind) => { + expect(classifyWorkspaceRestorationError(gitError(stderr))).toBe(kind); + }); + + it("classifies timeouts, typed connection failures, filesystem errors and checks", () => { + expect(classifyWorkspaceRestorationError(gitError("", { killed: true, signal: "SIGTERM" }))).toBe("timeout"); + expect(classifyWorkspaceRestorationError(new GitConnectionFailureError("clone failed", + { schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" }))).toBe("auth_rejected"); + expect(classifyWorkspaceRestorationError(new GitConnectionFailureError("clone failed", + { schemaVersion: 1, provider: "git", operation: "clone", reason: "dns_failure" }))).toBe("remote_unavailable"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("EACCES: permission denied, open '/x'"), { code: "EACCES" }))) + .toBe("filesystem"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("ENOSPC: no space left"), { code: "ENOSPC" }))).toBe("filesystem"); + expect(classifyWorkspaceRestorationError(new GitCacheRestorationFailure())).toBe("verification"); + expect(classifyWorkspaceRestorationError("not an error")).toBe("unknown"); + }); + + it("keeps the innermost restoration step and never carries Git output", () => { + const inner = new GitCacheRestorationFailure({ step: "saved_commit", kind: "commit_unavailable" }); + expect(workspaceReopenDiagnostic("source_verify", inner)).toEqual({ step: "saved_commit", kind: "commit_unavailable" }); + const diagnostic = workspaceReopenDiagnostic("managed_base", gitError("fatal: Authentication failed for 'https://user:sk-secret@example.test/r.git/'")); + expect(diagnostic).toEqual({ step: "managed_base", kind: "auth_rejected" }); + expect(JSON.stringify(diagnostic)).not.toContain("sk-secret"); + expect(workspaceReopenDiagnostic("publish", undefined)).toEqual({ step: "publish", kind: "unknown" }); + expect(inner.message).toBe("The saved workspace commit could not be restored. Check repository access and retry."); + }); + + it("bounds command text and distinguishes network failures from filesystem failures", () => { + const text = "x".repeat(16_384) + "Authentication failed for a private repository"; + expect(classifyWorkspaceRestorationError(gitError(text))).toBe("git_command"); + expect(classifyWorkspaceRestorationError(gitError("", { stderr: Buffer.from(text) }))).toBe("git_command"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("network failure"), { code: "ECONNRESET" }))).toBe("remote_unavailable"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("resolver failure"), { code: "EAI_AGAIN" }))).toBe("remote_unavailable"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("timeout"), { name: "TimeoutError" }))).toBe("timeout"); + expect(classifyWorkspaceRestorationError(gitError("", { code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", killed: true }))).toBe("git_command"); + expect(classifyWorkspaceRestorationError(Object.assign(new Error("unknown"), { code: "EUNRECOGNIZED" }))).toBe("unknown"); + }); +}); diff --git a/server/src/__tests__/issue-closed-workspace-routes.test.ts b/server/src/__tests__/issue-closed-workspace-routes.test.ts index e54a939d44d..9d109d1a63f 100644 --- a/server/src/__tests__/issue-closed-workspace-routes.test.ts +++ b/server/src/__tests__/issue-closed-workspace-routes.test.ts @@ -343,6 +343,7 @@ describe("closed isolated workspace issue routes", () => { ok: false, code: "rebuild_failed", message, + diagnostic: { step: "saved_commit", kind: "commit_unavailable" }, }); const app = request(createApp()); @@ -351,7 +352,7 @@ describe("closed isolated workspace issue routes", () => { : await app.patch(`/api/issues/${issueId}`).send({ comment: "continue" }); expect(res.status).toBe(409); - expect(res.body.error).toBe(message); + expect(res.body).toEqual({ error: message, diagnostic: { step: "saved_commit", kind: "commit_unavailable" } }); expect(mockIssueService.addComment).not.toHaveBeenCalled(); expect(mockIssueService.update).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/managed-repository-checkout.test.ts b/server/src/__tests__/managed-repository-checkout.test.ts index 2767407786e..c00957da6f4 100644 --- a/server/src/__tests__/managed-repository-checkout.test.ts +++ b/server/src/__tests__/managed-repository-checkout.test.ts @@ -4,7 +4,7 @@ import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, symlink } from import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored } from "../services/managed-repository-checkout.js"; +import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored, readManagedRepositoryPreparationFailure } from "../services/managed-repository-checkout.js"; const execFile = promisify(execFileCallback); const directories: string[] = []; @@ -61,6 +61,29 @@ describe("atomic managed repository publication", () => { await expect(ensureManagedRepositoriesIgnored(target, { plainTaskDirectory: true })).rejects.toThrow("must not be a symlink"); expect(await readFile(path.join(source, ".git", "info", "exclude"), "utf8")).toBe(original); }); + it.each([ + { label: "timeout", code: null, killed: true, timedOut: true }, + { label: "filesystem", code: "EACCES", killed: false, timedOut: false }, + { label: "output limit", code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", killed: true, timedOut: false }, + ])("retains safe $label metadata without adding raw command causes", async ({ code, killed, timedOut }) => { + const { source, target } = await fixture(); + const original = Object.assign(new Error("Git failed for https://user:fixture-password@example.test/repo.git"), { + code, killed, stderr: "private command output", + }); + const failure = await materializeManagedProjectWorkspace(target, { + repoUrl: "https://example.test/repo.git", localSource: source, + beforePublish: async () => { throw original; }, + }).then(() => { throw new Error("Expected preparation to fail"); }, (error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(readManagedRepositoryPreparationFailure(failure)).toEqual({ code, timedOut }); + expect(failure).not.toHaveProperty("code"); + expect(failure).not.toHaveProperty("cause"); + expect(failure).not.toHaveProperty("stderr"); + expect(JSON.stringify(failure)).toBe("{}"); + expect((failure as Error).message).not.toContain("fixture-password"); + expect(await stat(target).catch(() => null)).toBeNull(); + }); + it("writes the pinned receipt inside the clone before making it visible", async () => { const { source, target } = await fixture(); const result = await materializeManagedProjectWorkspace(target, { repoUrl: "https://github.com/test/source", localSource: source, beforePublish: async clone => { diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 0a1ce1705b9..4bf0e443318 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -8014,7 +8014,7 @@ export function issueRoutes( "This issue is linked to a closed workspace that cannot be reopened.", }); } else { - res.status(409).json({ error: result.message }); + res.status(409).json({ error: result.message, ...(result.diagnostic ? { diagnostic: result.diagnostic } : {}) }); } return null; } diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index ab9462e811e..4ee7b3daa5f 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -20,7 +20,8 @@ import { hasPendingNestedCloneMigration, isNestedCloneMigrationCandidate, nextNe NESTED_CLONE_MIGRATION_RETRY_REASONS, type NestedCloneMigrationRotation } from "./nested-clone-migration.js"; import type { StorageService } from "../storage/types.js"; import { getStorageService } from "../storage/index.js"; -import { readGitCacheRestoration, restoreGitCacheBase, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, gitRestorePointExpired } from "./git-workspace-restoration.js"; +import { readGitCacheRestoration, restoreGitCacheBase, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, gitRestorePointExpired, + classifyWorkspaceRestorationError, type GitCacheRestorationStep, type WorkspaceRestorationFailureKind } from "./git-workspace-restoration.js"; export const GIT_RESTORE_POINT_EXPIRED_WARNING = "The saved state of this workspace expired, so it was reopened as a fresh workspace on its branch. Unpushed and uncommitted work from before it was cleaned up is no longer available."; import { parseObject } from "../adapters/utils.js"; @@ -327,7 +328,16 @@ export type ReopenClosedIsolatedExecutionWorkspaceResult = // generation, so a stale actor never clears a newer reopen's fence. | { ok: true; workspace: ExecutionWorkspace; reopened: true; generation: number; warning?: string } | { ok: true; workspace: ExecutionWorkspace; reopened: false; generation: number } - | { ok: false; code: "not_reopenable" | "rebuild_failed"; message: string }; + | { ok: false; code: "not_reopenable" | "rebuild_failed"; message: string; diagnostic?: WorkspaceReopenDiagnostic }; + +export type WorkspaceReopenFailureStep = GitCacheRestorationStep | "managed_base" | "copy_identity" | "expiration_locks" + | "workspace_rebuild" | "publish" | "transaction"; +export type WorkspaceReopenDiagnostic = { step: WorkspaceReopenFailureStep; kind: WorkspaceRestorationFailureKind }; + +export function workspaceReopenDiagnostic(step: WorkspaceReopenFailureStep, error: unknown): WorkspaceReopenDiagnostic { + if (error instanceof GitCacheRestorationFailure && error.step) return { step: error.step, kind: classifyWorkspaceRestorationError(error) }; + return { step, kind: error === undefined ? "unknown" : classifyWorkspaceRestorationError(error) }; +} export type ExecutionWorkspaceServiceOptions = { pluginWorkerManager?: WorkspaceWorkers; @@ -4345,9 +4355,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); } await assertTaskWorkspaceExecutionAccess(db, input.authorizationActor, issue.companyId, { issueId: issue.id, workspaceId: snapshot.id }); - let recipe: ReturnType; + let recipe: ReturnType = null; + const rebuildFailed = (step: WorkspaceReopenFailureStep, error: unknown, restoration = recipe !== null): ReopenClosedIsolatedExecutionWorkspaceResult => { + const diagnostic = workspaceReopenDiagnostic(step, error); + logger.warn({ event: "execution_workspace.reopen", outcome: "rebuild_failed", executionWorkspaceId: snapshot.id, + issueId: issue.id, companyId: issue.companyId, ...diagnostic }, "execution workspace reopen rebuild failed"); + return { ok: false, code: "rebuild_failed", diagnostic, + message: restoration ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + }; try { recipe = readGitCacheRestoration(snapshot); } - catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; } + catch (error) { return rebuildFailed("recipe", error, true); } const freshStart = recipe !== null && isClosedExecutionWorkspaceStatus(snapshot.status) && gitRestorePointExpired(recipe); const gitAuth = createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id); const [ @@ -4383,9 +4400,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic resolvedBaseCwd = recipe && isClosedExecutionWorkspaceStatus(snapshot.status) ? await restoreGitCacheBase(snapshot, recipe, gitAuth, undefined, { freshStart }) : await verifiedRecordedIsolatedRemoteSource(snapshot) ?? projectWorkspace?.cwd ?? null; - } catch { - return { ok: false, code: "rebuild_failed", message: recipe - ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + } catch (error) { + return rebuildFailed("source_verify", error); } if (isClosedExecutionWorkspaceStatus(snapshot.status) && resolvedBaseCwd == null && snapshot.strategyType === "git_worktree" && snapshot.projectId) { try { @@ -4409,8 +4425,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic resolveGitAuth: createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id), }); resolvedBaseCwd = managedWorkspace.cwd; - } catch { - return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + } catch (error) { + return rebuildFailed("managed_base", error); } } let copyIdentity: { sourceCwd: string; configuredSourceCwd: string; targetCwd: string } | null = null; @@ -4427,8 +4443,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic sourceCwd: await fs.realpath(sourceRoot), configuredSourceCwd: await fs.realpath(resolvedBaseCwd), targetCwd, }; - } catch { - return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + } catch (error) { + return rebuildFailed("copy_identity", error); } } const result = await db.transaction(async (tx): Promise => { @@ -4478,7 +4494,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } try { await recoverGitCacheExpirationLocks(row); } - catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; } + catch (error) { return rebuildFailed("expiration_locks", error, true); } const verifyCopyIdentity = copyIdentity ? async (sourceCwd: string, targetCwd: string) => { if (await fs.realpath(sourceCwd) !== copyIdentity.sourceCwd || await fs.realpath(resolvedBaseCwd!) !== copyIdentity.configuredSourceCwd @@ -4525,6 +4541,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); let rebuildError: string | null = null; + let rebuildDiagnostic: WorkspaceReopenDiagnostic | null = null; let taskDirectoryWarning: string | null = null; const taskDirectoryState = readTaskDirectoryRetention(row); try { @@ -4644,9 +4661,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); if (!realized) { rebuildError = "Execution workspace could not be rebuilt"; + rebuildDiagnostic = workspaceReopenDiagnostic("workspace_rebuild", undefined); } } catch (error) { rebuildError = error instanceof Error ? error.message : String(error); + rebuildDiagnostic = workspaceReopenDiagnostic("workspace_rebuild", error); } if (rebuildError) { @@ -4677,6 +4696,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic actorType: actor.actorType, actorAgentId: actor.agentId ?? null, generation: nextGeneration, + ...rebuildDiagnostic, error: rebuildError, }, "execution workspace reopen rebuild failed", @@ -4693,9 +4713,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic issueId: issue.id, outcome: "rebuild_failed", generation: nextGeneration, + ...rebuildDiagnostic, }, }); - return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + return { ok: false, code: "rebuild_failed", ...(rebuildDiagnostic ? { diagnostic: rebuildDiagnostic } : {}), + message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; } // The rebuild succeeded. Publish the row as active in one write, and clear @@ -4730,7 +4752,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic .returning() .then((rows) => rows[0] ?? null); if (!activeRow) { - return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + return rebuildFailed("publish", undefined); } logger.info( { @@ -4770,7 +4792,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic ...(reopenWarning ? { warning: reopenWarning } : {}) }; }).catch((error): ReopenClosedIsolatedExecutionWorkspaceResult => { logger.warn({ error, executionWorkspaceId: snapshot.id }, "Execution workspace reopen failed"); - return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" }; + return rebuildFailed("transaction", error); }); return result; }; diff --git a/server/src/services/git-workspace-contents.ts b/server/src/services/git-workspace-contents.ts index 9e138816044..1e8e8a74774 100644 --- a/server/src/services/git-workspace-contents.ts +++ b/server/src/services/git-workspace-contents.ts @@ -10,7 +10,7 @@ import type { GitRemoteAuthProvider } from "./git-credentials.js"; import { localRepositoryPath } from "./local-repository-path.js"; import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; -import { ensureSavedGitCommit, gitCacheCommand, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; +import { ensureSavedGitCommit, gitCacheCommand, GitCacheRestorationFailure, restorationFailureAt } from "./git-workspace-restoration.js"; import { applyGitRestorePointToWorktree, ensureGitRestorePointCommit, loadGitRestorePoint, type GitRestorePointRef } from "./git-restore-point-storage.js"; import { copyVerifiedGitExcludes } from "./git-repository-excludes.js"; import { importRestorePoint } from "./git-restore-point.js"; @@ -291,30 +291,36 @@ export async function restoreGitWorkspaceContents(input: Identity & { cwd: strin } const savedRestorePoint = checkout.restorePoint && input.isolateRepositories ? await loadGitRestorePoint({ storage: input.storage ?? getStorageService(), workspace, ref: checkout.restorePoint }) : null; + let preparationFailure: GitCacheRestorationFailure | null = null; const result = await materializeManagedProjectWorkspace(destination, { repoUrl: checkout.source.repoUrl, gitObjectSource: gitObjectSource ? { cwd: gitObjectSource, commits: [checkout.commit, ...(savedRestorePoint?.manifest.prerequisites ?? [])] } : undefined, resolveGitAuth: input.resolveGitAuth, beforePublish: async temporary => { - if (gitObjectSource && checkout.restorePoint) await copyVerifiedGitExcludes(gitObjectSource, temporary); - if (savedRestorePoint) { - const { payload, manifest } = savedRestorePoint; - await ensureCheckoutPrerequisites(temporary, checkout, manifest, input.resolveGitAuth); - await importRestorePoint({ repositoryCwd: temporary, payload }); - } else if (checkout.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage ?? getStorageService(), workspace, - ref: checkout.restorePoint, repositoryCwd: temporary }); - await ensureSavedGitCommit(temporary, checkout.source.repoUrl, checkout.commit, input.resolveGitAuth); - if (checkout.branchName) await gitCacheCommand(temporary, ["check-ref-format", "--branch", checkout.branchName]); - await gitCacheCommand(temporary, checkout.branchName - ? ["checkout", "-B", checkout.branchName, checkout.commit] : ["checkout", "--detach", checkout.commit]); - if (checkout.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage ?? getStorageService(), workspace, - ref: checkout.restorePoint, repositoryCwd: temporary, worktreePath: temporary, preserveImportedHistory: input.isolateRepositories }); - await gitCacheCommand(temporary, ["reflog", "expire", "--expire=now", "--all"]); - await restoreSettings(temporary, checkout.settings, input, relative); - await input.authorizeSource(checkout.source); - await recheck?.(); - if (gitObjectSource && !await isVerifiedNestedRepositoryBase(gitObjectSource, candidates!, checkout.source.repoUrl)) throw new GitCacheRestorationFailure(); - } }); + try { + if (gitObjectSource && checkout.restorePoint) await copyVerifiedGitExcludes(gitObjectSource, temporary); + if (savedRestorePoint) { + const { payload, manifest } = savedRestorePoint; + await ensureCheckoutPrerequisites(temporary, checkout, manifest, input.resolveGitAuth); + await importRestorePoint({ repositoryCwd: temporary, payload }); + } else if (checkout.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage ?? getStorageService(), workspace, + ref: checkout.restorePoint, repositoryCwd: temporary }); + await ensureSavedGitCommit(temporary, checkout.source.repoUrl, checkout.commit, input.resolveGitAuth); + if (checkout.branchName) await gitCacheCommand(temporary, ["check-ref-format", "--branch", checkout.branchName]); + await gitCacheCommand(temporary, checkout.branchName + ? ["checkout", "-B", checkout.branchName, checkout.commit] : ["checkout", "--detach", checkout.commit]); + if (checkout.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage ?? getStorageService(), workspace, + ref: checkout.restorePoint, repositoryCwd: temporary, worktreePath: temporary, preserveImportedHistory: input.isolateRepositories }); + await gitCacheCommand(temporary, ["reflog", "expire", "--expire=now", "--all"]); + await restoreSettings(temporary, checkout.settings, input, relative); + await input.authorizeSource(checkout.source); + await recheck?.(); + if (gitObjectSource && !await isVerifiedNestedRepositoryBase(gitObjectSource, candidates!, checkout.source.repoUrl)) throw new GitCacheRestorationFailure(); + } catch (error) { + preparationFailure = restorationFailureAt("workspace_contents", error); + throw preparationFailure; + } + } }).catch(error => { throw preparationFailure ?? error; }); if (result.warning) throw new GitCacheRestorationFailure(); } await restoreSettings(input.cwd, input.contents.settings, input, CLAUDE_SETTINGS_PATH); - } catch { throw new GitCacheRestorationFailure(); } + } catch (error) { throw restorationFailureAt("workspace_contents", error); } } diff --git a/server/src/services/git-workspace-restoration.ts b/server/src/services/git-workspace-restoration.ts index b84325b0d83..2039039f03d 100644 --- a/server/src/services/git-workspace-restoration.ts +++ b/server/src/services/git-workspace-restoration.ts @@ -8,7 +8,8 @@ import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; import { createHash } from "node:crypto"; import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js"; -import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; +import { materializeManagedProjectWorkspace, RepositoryRefNotFoundError, readManagedRepositoryPreparationFailure } from "./managed-repository-checkout.js"; +import { readGitConnectionFailure } from "./git-connection-failure.js"; import { promisify } from "node:util"; import type { GitRemoteAuthProvider } from "./git-credentials.js"; import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js"; @@ -85,8 +86,63 @@ export async function verifiedManagedGitObjectSource(workspace: Workspace, sourc return sourcePath; } +export type GitCacheRestorationStep = "recipe" | "trash_recovery" | "source_check" | "source_materialize" | "source_verify" + | "restore_point" | "saved_commit" | "workspace_contents"; +export type WorkspaceRestorationFailureKind = "remote_unavailable" | "auth_rejected" | "commit_unavailable" | "timeout" + | "verification" | "filesystem" | "git_command" | "unknown"; + export class GitCacheRestorationFailure extends Error { - constructor() { super("The saved workspace commit could not be restored. Check repository access and retry."); } + readonly step: GitCacheRestorationStep | null; + readonly kind: WorkspaceRestorationFailureKind | null; + constructor(diagnostic: { step: GitCacheRestorationStep; kind: WorkspaceRestorationFailureKind } | null = null) { + super("The saved workspace commit could not be restored. Check repository access and retry."); + this.step = diagnostic?.step ?? null; + this.kind = diagnostic?.kind ?? null; + } +} + +export function classifyWorkspaceRestorationError(error: unknown): WorkspaceRestorationFailureKind { + if (error instanceof GitCacheRestorationFailure) return error.kind ?? "verification"; + if (!(error instanceof Error)) return "unknown"; + const connection = readGitConnectionFailure(error); + if (connection) { + if (connection.reason === "authentication_failed") return "auth_rejected"; + return connection.reason === "connection_timeout" ? "timeout" : "remote_unavailable"; + } + if (error instanceof RepositoryRefNotFoundError) return "commit_unavailable"; + const details = error as Error & { code?: unknown; killed?: unknown; stderr?: unknown }; + const preparationFailure = readManagedRepositoryPreparationFailure(error); + const commandCode = preparationFailure?.code ?? details.code; + const code = typeof commandCode === "string" ? commandCode : null; + if (commandCode === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") return "git_command"; + if (preparationFailure?.timedOut || details.killed === true || commandCode === "ETIMEDOUT" || error.name === "AbortError" || error.name === "TimeoutError") return "timeout"; + if (["ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "ECONNRESET", "EHOSTUNREACH", "ENETUNREACH"].includes(code ?? "")) return "remote_unavailable"; + if (["ENOENT", "ENOTDIR", "EISDIR", "EACCES", "EPERM", "ENOSPC", "EROFS", "EMFILE", "ENFILE", "ELOOP", "ENAMETOOLONG", "EBUSY", "EIO", "EDQUOT"].includes(code ?? "")) return "filesystem"; + const stderr = typeof details.stderr === "string" ? details.stderr.slice(0, 16_384) + : Buffer.isBuffer(details.stderr) ? details.stderr.subarray(0, 16_384).toString("utf8") : error.message.slice(0, 16_384); + const lines = stderr.split(/\r?\n/).map(line => line.trim()); + if (lines.some(line => /^(?:error|fatal): (?:cannot|could not|unable to) (?:open|create|write|lock|read|stat|mkdir|unlink|remove|rename)\b.*: (?:Permission denied|Operation not permitted|Read-only file system|No space left on device)\.?$/i.test(line) + || /^(?:error|fatal): unable to access ['"](?![a-z][a-z0-9+.-]*:\/\/)[^'"\r\n]+['"]: (?:Permission denied|Operation not permitted|Read-only file system)\.?$/i.test(line))) return "filesystem"; + if (lines.some(line => /^fatal: Authentication failed for ['"][^'"\r\n]+['"]$/i.test(line) + || /^fatal: could not read (?:Username|Password) for ['"][^'"\r\n]+['"]: terminal prompts disabled$/i.test(line) + || /^fatal: unable to access ['"][^'"\r\n]+['"]: The requested URL returned error: 40[13]$/i.test(line) + || /^remote: (?:Invalid username or (?:password|token)|Permission [^\r\n]+ denied|Support for password authentication was removed|Password authentication is not supported)/i.test(line) + || /^(?:[^\s:@]+@)?[a-z0-9][a-z0-9.-]*: Permission denied \((?:publickey|password|keyboard-interactive)(?:,[a-z-]+)*\)\.?$/i.test(line))) { + return "auth_rejected"; + } + if (/not our ref|couldn't find remote ref|no such remote ref|unadvertised object|did not send all necessary objects/i.test(stderr)) { + return "commit_unavailable"; + } + if (/could not resolve host|unable to access|could not read from remote repository|does not appear to be a git repository|repository .*not found|connection (refused|timed out|reset)/i.test(stderr)) { + return "remote_unavailable"; + } + if (typeof details.stderr === "string" || typeof commandCode === "number") return "git_command"; + return "unknown"; +} + +export function restorationFailureAt(step: GitCacheRestorationStep, error: unknown) { + if (error instanceof GitCacheRestorationFailure && error.step) return error; + return new GitCacheRestorationFailure({ step, kind: classifyWorkspaceRestorationError(error) }); } export function readGitCacheRestoration(workspace: Workspace): GitCacheRestoration | null { @@ -719,9 +775,12 @@ export function gitRestorePointExpired(recipe: GitCacheRestoration) { export async function restoreGitCacheBase(workspace: Workspace, recipe: GitCacheRestoration, resolveGitAuth: GitRemoteAuthProvider, storage?: StorageService, options: { freshStart?: boolean } = {}) { - if (readGitCacheRestoration(workspace)?.commit !== recipe.commit) throw new GitCacheRestorationFailure(); + let step: GitCacheRestorationStep = "recipe"; try { + if (readGitCacheRestoration(workspace)?.commit !== recipe.commit) throw new GitCacheRestorationFailure(); + step = "trash_recovery"; await finishGitCacheTrash(workspace); + step = "source_check"; const present = await fs.lstat(recipe.sourcePath).then(() => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; @@ -736,21 +795,25 @@ export async function restoreGitCacheBase(workspace: Workspace, recipe: GitCache if (!entry.isDirectory() || entry.isSymbolicLink() || await fs.realpath(parent) !== parent) throw new GitCacheRestorationFailure(); } } + step = present ? "source_verify" : "source_materialize"; const cwd = present ? await verifiedGitCacheSource(workspace, recipe.sourcePath) : recipe.sourceKind === "isolated_remote" ? await materializeIsolatedRemoteWorkspaceSource({ companyId: recipe.companyId, issueId: recipe.sourceIssueId, repoUrl: recipe.repoUrl, resolveGitAuth }) : (await materializeManagedProjectWorkspace(recipe.sourcePath, { repoUrl: recipe.repoUrl, resolveGitAuth })).cwd; + step = "source_verify"; await verifiedGitCacheSource(workspace, recipe.sourcePath); if (!cwd) throw new GitCacheRestorationFailure(); if (options.freshStart) return cwd; + step = "restore_point"; if (recipe.restorePoint) { await ensureGitRestorePointCommit({ storage: storage ?? getStorageService(), workspace: { companyId: workspace.companyId, id: workspace.id }, ref: recipe.restorePoint, repositoryCwd: cwd }); } + step = "saved_commit"; await ensureSavedGitCommit(cwd, recipe.repoUrl, recipe.commit, resolveGitAuth); return cwd; - } catch { throw new GitCacheRestorationFailure(); } + } catch (error) { throw restorationFailureAt(step, error); } } export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit: string, resolveGitAuth: (url: string) => Promise<{ configArgs: string[]; env: Record } | null>) { @@ -762,7 +825,7 @@ export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit: timeout: 60_000, maxBuffer: 1024 * 1024, env: { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0" }, }); if ((await gitCacheCommand(cwd, ["rev-parse", `${commit}^{commit}`])).trim() !== commit) throw new GitCacheRestorationFailure(); - } catch { throw new GitCacheRestorationFailure(); } + } catch (error) { throw restorationFailureAt("saved_commit", error); } } /** Runtime callers must bind the recipe to the selected workspace and base. */ diff --git a/server/src/services/managed-repository-checkout.ts b/server/src/services/managed-repository-checkout.ts index cbd8ad0b1d2..73a452733fe 100644 --- a/server/src/services/managed-repository-checkout.ts +++ b/server/src/services/managed-repository-checkout.ts @@ -15,6 +15,29 @@ import { resolveRepositoryObjectCache, readRepositoryObjectCache, publishReposit const execFile = promisify(execFileCallback); const MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS = 10 * 60 * 1000; +type ManagedRepositoryPreparationFailure = { code: string | number | null; timedOut: boolean }; +const preparationFailures = new WeakMap(); + +export function readManagedRepositoryPreparationFailure(error: unknown): ManagedRepositoryPreparationFailure | null { + const failure = error instanceof Error ? preparationFailures.get(error) : undefined; + return failure ? { ...failure } : null; +} + +function managedRepositoryPreparationError(message: string, error: unknown): Error { + const wrapped = new Error(message); + const retained = readManagedRepositoryPreparationFailure(error); + if (retained) preparationFailures.set(wrapped, retained); + else if (error instanceof Error) { + const details = error as Error & { code?: unknown; killed?: unknown }; + const code = typeof details.code === "number" && Number.isSafeInteger(details.code) ? details.code + : typeof details.code === "string" && /^(?:E[A-Z0-9_]{1,63}|ABORT_ERR)$/.test(details.code) ? details.code : null; + const timedOut = code !== "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" && (details.killed === true || code === "ETIMEDOUT" + || error.name === "AbortError" || error.name === "TimeoutError"); + if (code !== null || timedOut) preparationFailures.set(wrapped, { code, timedOut }); + } + return wrapped; +} + async function copyManagedGitObjects(cwd: string, repoUrl: string, source: { cwd: string; commits: readonly string[] }) { const git = (directory: string, args: string[]) => execFile("git", ["-C", directory, ...args], { env: { ...sanitizeRuntimeServiceBaseEnv(process.env), GIT_TERMINAL_PROMPT: "0" }, @@ -202,7 +225,7 @@ export async function materializeManagedProjectWorkspace( if (error instanceof RepositoryRefNotFoundError) throw new RepositoryRefNotFoundError(message, error.requestedRef); const connectionFailure = readGitConnectionFailure(error); if (connectionFailure) throw new GitConnectionFailureError(message, connectionFailure); - throw new Error(message); + throw managedRepositoryPreparationError(message, error); } try { @@ -216,8 +239,9 @@ export async function materializeManagedProjectWorkspace( } const reason = renameError instanceof Error ? renameError.message : String(renameError); - throw new Error( + throw managedRepositoryPreparationError( `Failed to move managed checkout into place at "${cwd}": ${reason}`, + renameError, ); } return { cwd, warning: null };