fix(workspaces): report safe Git restore diagnostics (#19736)

Return the restore stage and failure category with the existing HTTP409 rejection. Preserve the unsaved comment and editable draft. Keep nested saved-commit and clone-timeout diagnostics through preparation wrappers, and distinguish local Git file errors from authentication failures. Retain only safe process flags privately; no raw command cause is exposed.

All seven affected local suites pass (288 tests) and the layered server typecheck passes on supported Node25.6.1. Reviewed head 022eb7c2eb has Greptile Apex5/5, zero unresolved findings and all applicable CI passing (55 successful checks and4 intentional skips).

Co-Authored-By: Sentry Bot <sentry-bot@paperclip.invalid>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
authored and GitHub committed 2026-11-06 21:33:59 -08:00
1 parent 464e6e16e0
commit 131b566ca4
10 files changed
+325 -47

No files matched your search

+10
View File
@@ -4533,6 +4533,16 @@ retryable restoration failure. It never substitutes the default branch.
Reopening failures reject the consuming request with HTTP 409 before saving a
comment or changing its task. The response preserves the restoration message, so
the composer keeps an editable draft and permits retry after access is restored.
It also carries `diagnostic: { step, kind }`, and the server log records the same
pair. Both are closed values: `step` names the failed stage (for example
`source_materialize`, `restore_point` or `saved_commit`), and `kind` names the
failure class (`remote_unavailable`, `auth_rejected`, `commit_unavailable`,
`timeout`, `verification`, `filesystem`, `git_command` or `unknown`). They never
contain paths, repository URLs or Git output.
Nested checkout failures retain their original restore step. The checkout
wrapper keeps process error codes and timeout flags privately for classification;
it does not attach the raw command error as a cause or add those flags to the
public error object.
Both enable switches default off. Rollback disables further expiration while
leaving saved recipes and restoration support intact. Recovery archives, runner
@@ -28,6 +28,21 @@ import * as activityLog from "../services/activity-log.js";
import { readGitCacheRestoration } from "../services/git-workspace-restoration.js";
import { assertReproducibleManagedCheckout, gitCacheExpirationLockLocations, readGitCacheExpiration } from "../services/git-workspace-restoration.js";
const cloneFailure = vi.hoisted(() => ({ error: null as Error | null }));
vi.mock("node:child_process", async importOriginal => {
const actual = await importOriginal<typeof import("node:child_process")>();
const { promisify } = await import("node:util");
const execute = promisify(actual.execFile);
const execFile = (...args: unknown[]) => Reflect.apply(actual.execFile, undefined, args);
Object.defineProperty(execFile, promisify.custom, { value: (...args: unknown[]) => {
if (cloneFailure.error && args[0] === "git" && Array.isArray(args[1]) && args[1].includes("clone")) {
return Promise.reject(cloneFailure.error);
}
return Reflect.apply(execute, undefined, args);
} });
return { ...actual, execFile };
});
const gitEnv = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test",
GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test", GIT_TERMINAL_PROMPT: "0" };
const git = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, env: gitEnv, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
@@ -986,13 +1001,68 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => {
const f = await fixture();
expect((await f.expire()).baseExpired).toBe(true);
await fs.rename(f.remote, `${f.remote}.offline`);
expect(await f.reopen()).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry." });
const failed = await f.reopen();
expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry.",
diagnostic: { step: "source_materialize", kind: "remote_unavailable" } });
expect(JSON.stringify(failed)).not.toContain(root);
expect(await f.reopen(randomUUID())).toMatchObject({ ok: false, code: "not_reopenable" });
expect(await exists(f.cwd)).toBe(false);
await fs.rename(`${f.remote}.offline`, f.remote);
expect(await f.reopen()).toMatchObject({ ok: true });
expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA);
});
it("reports the failed step and kind when the saved commit is gone from upstream", async () => {
const f = await fixture();
expect((await f.expire()).baseExpired).toBe(true);
git(f.source, "checkout", "--orphan", "rewritten");
await fs.writeFile(path.join(f.source, "marker.txt"), "rewritten\n");
git(f.source, "add", "."); git(f.source, "commit", "-m", "Rewritten");
git(f.source, "push", "--force", "origin", "rewritten:main");
git(f.remote, "reflog", "expire", "--expire=now", "--all"); git(f.remote, "gc", "--prune=now");
const failed = await f.reopen();
expect(failed).toMatchObject({ ok: false, code: "rebuild_failed", message: "The saved workspace commit could not be restored. Check repository access and retry.",
diagnostic: { step: "saved_commit", kind: "commit_unavailable" } });
expect(JSON.stringify(failed)).not.toContain(root);
expect(JSON.stringify(failed)).not.toContain(f.commitA);
expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ commit: f.commitA });
});
it("retains the nested saved commit failure through checkout preparation and reopening", async () => {
const f = await nestedFixture(true, false);
expect((await f.expire({ expireBase: false })).expired).toBe(true);
git(f.nestedSource, "checkout", "--orphan", "rewritten");
await fs.writeFile(path.join(f.nestedSource, "nested.txt"), "rewritten\n");
git(f.nestedSource, "add", "."); git(f.nestedSource, "commit", "-m", "Rewritten nested history");
git(f.nestedSource, "push", "--force", f.nestedRemote, "rewritten:main");
git(f.nestedRemote, "reflog", "expire", "--expire=now", "--all");
git(f.nestedRemote, "gc", "--prune=now");
const failed = await f.reopen();
expect(failed).toMatchObject({ ok: false, code: "rebuild_failed",
diagnostic: { step: "saved_commit", kind: "commit_unavailable" } });
expect(JSON.stringify(failed)).not.toContain(root);
expect(JSON.stringify(failed)).not.toContain(f.nestedA);
expect((await f.row()).metadata?.gitCacheRestoration).toMatchObject({ contents: {
checkouts: [expect.objectContaining({ commit: f.nestedA })],
} });
});
it("retains a managed clone timeout through preparation and reopening", async () => {
const f = await fixture(true, false);
expect((await f.expire({ expireBase: false })).expired).toBe(true);
await fs.rename(f.baseCwd, `${f.baseCwd}.offline`);
cloneFailure.error = Object.assign(new Error("clone timed out; fixture credential must stay private"), {
killed: true, signal: "SIGTERM", code: null,
});
let failed: Awaited<ReturnType<typeof f.reopen>>;
try { failed = await f.reopen(); }
finally { cloneFailure.error = null; }
expect(failed).toMatchObject({ ok: false, code: "rebuild_failed",
diagnostic: { step: "source_materialize", kind: "timeout" } });
expect(JSON.stringify(failed)).not.toContain(root);
expect(JSON.stringify(failed)).not.toContain("fixture credential");
expect(await f.reopen()).toMatchObject({ ok: true });
});
it("serializes concurrent reopening after deletion and publishes one transition", async () => {
const f = await fixture();
let release!: () => void, entered!: () => void;
@@ -0,0 +1,59 @@
import { describe, expect, it } from "vitest";
import { classifyWorkspaceRestorationError, GitCacheRestorationFailure } from "../services/git-workspace-restoration.js";
import { workspaceReopenDiagnostic } from "../services/execution-workspaces.js";
import { GitConnectionFailureError } from "../services/git-connection-failure.js";
const gitError = (stderr: string, extra: Record<string, unknown> = {}) =>
Object.assign(new Error(`Command failed: git fetch\n${stderr}`), { code: 128, stderr, ...extra });
describe("workspace restoration diagnostic", () => {
it.each([
["fatal: Authentication failed for 'https://user:sk-secret@example.test/repo.git/'", "auth_rejected"],
["fatal: could not read Username for 'https://github.com': terminal prompts disabled", "auth_rejected"],
["fatal: remote error: upload-pack: not our ref 9a204a56", "commit_unavailable"],
["fatal: couldn't find remote ref refs/heads/gone", "commit_unavailable"],
["fatal: unable to access 'https://example.test/repo.git/': Could not resolve host: example.test", "remote_unavailable"],
["fatal: '/srv/repo.git' does not appear to be a git repository", "remote_unavailable"],
["fatal: bad revision", "git_command"],
["error: cannot open '.git/FETCH_HEAD': Permission denied", "filesystem"],
["fatal: unable to access '/private/repository/.git/config': Permission denied", "filesystem"],
["git@example.test: Permission denied (publickey).", "auth_rejected"],
["git@example.test: Permission denied (publickey).\nerror: cannot open '.git/FETCH_HEAD': Permission denied", "filesystem"],
])("classifies %s", (stderr, kind) => {
expect(classifyWorkspaceRestorationError(gitError(stderr))).toBe(kind);
});
it("classifies timeouts, typed connection failures, filesystem errors and checks", () => {
expect(classifyWorkspaceRestorationError(gitError("", { killed: true, signal: "SIGTERM" }))).toBe("timeout");
expect(classifyWorkspaceRestorationError(new GitConnectionFailureError("clone failed",
{ schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" }))).toBe("auth_rejected");
expect(classifyWorkspaceRestorationError(new GitConnectionFailureError("clone failed",
{ schemaVersion: 1, provider: "git", operation: "clone", reason: "dns_failure" }))).toBe("remote_unavailable");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("EACCES: permission denied, open '/x'"), { code: "EACCES" })))
.toBe("filesystem");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("ENOSPC: no space left"), { code: "ENOSPC" }))).toBe("filesystem");
expect(classifyWorkspaceRestorationError(new GitCacheRestorationFailure())).toBe("verification");
expect(classifyWorkspaceRestorationError("not an error")).toBe("unknown");
});
it("keeps the innermost restoration step and never carries Git output", () => {
const inner = new GitCacheRestorationFailure({ step: "saved_commit", kind: "commit_unavailable" });
expect(workspaceReopenDiagnostic("source_verify", inner)).toEqual({ step: "saved_commit", kind: "commit_unavailable" });
const diagnostic = workspaceReopenDiagnostic("managed_base", gitError("fatal: Authentication failed for 'https://user:sk-secret@example.test/r.git/'"));
expect(diagnostic).toEqual({ step: "managed_base", kind: "auth_rejected" });
expect(JSON.stringify(diagnostic)).not.toContain("sk-secret");
expect(workspaceReopenDiagnostic("publish", undefined)).toEqual({ step: "publish", kind: "unknown" });
expect(inner.message).toBe("The saved workspace commit could not be restored. Check repository access and retry.");
});
it("bounds command text and distinguishes network failures from filesystem failures", () => {
const text = "x".repeat(16_384) + "Authentication failed for a private repository";
expect(classifyWorkspaceRestorationError(gitError(text))).toBe("git_command");
expect(classifyWorkspaceRestorationError(gitError("", { stderr: Buffer.from(text) }))).toBe("git_command");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("network failure"), { code: "ECONNRESET" }))).toBe("remote_unavailable");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("resolver failure"), { code: "EAI_AGAIN" }))).toBe("remote_unavailable");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("timeout"), { name: "TimeoutError" }))).toBe("timeout");
expect(classifyWorkspaceRestorationError(gitError("", { code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", killed: true }))).toBe("git_command");
expect(classifyWorkspaceRestorationError(Object.assign(new Error("unknown"), { code: "EUNRECOGNIZED" }))).toBe("unknown");
});
});
@@ -343,6 +343,7 @@ describe("closed isolated workspace issue routes", () => {
ok: false,
code: "rebuild_failed",
message,
diagnostic: { step: "saved_commit", kind: "commit_unavailable" },
});
const app = request(createApp());
@@ -351,7 +352,7 @@ describe("closed isolated workspace issue routes", () => {
: await app.patch(`/api/issues/${issueId}`).send({ comment: "continue" });
expect(res.status).toBe(409);
expect(res.body.error).toBe(message);
expect(res.body).toEqual({ error: message, diagnostic: { step: "saved_commit", kind: "commit_unavailable" } });
expect(mockIssueService.addComment).not.toHaveBeenCalled();
expect(mockIssueService.update).not.toHaveBeenCalled();
});
@@ -4,7 +4,7 @@ import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, stat, symlink } from
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored } from "../services/managed-repository-checkout.js";
import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored, readManagedRepositoryPreparationFailure } from "../services/managed-repository-checkout.js";
const execFile = promisify(execFileCallback);
const directories: string[] = [];
@@ -61,6 +61,29 @@ describe("atomic managed repository publication", () => {
await expect(ensureManagedRepositoriesIgnored(target, { plainTaskDirectory: true })).rejects.toThrow("must not be a symlink");
expect(await readFile(path.join(source, ".git", "info", "exclude"), "utf8")).toBe(original);
});
it.each([
{ label: "timeout", code: null, killed: true, timedOut: true },
{ label: "filesystem", code: "EACCES", killed: false, timedOut: false },
{ label: "output limit", code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", killed: true, timedOut: false },
])("retains safe $label metadata without adding raw command causes", async ({ code, killed, timedOut }) => {
const { source, target } = await fixture();
const original = Object.assign(new Error("Git failed for https://user:fixture-password@example.test/repo.git"), {
code, killed, stderr: "private command output",
});
const failure = await materializeManagedProjectWorkspace(target, {
repoUrl: "https://example.test/repo.git", localSource: source,
beforePublish: async () => { throw original; },
}).then(() => { throw new Error("Expected preparation to fail"); }, (error: unknown) => error);
expect(failure).toBeInstanceOf(Error);
expect(readManagedRepositoryPreparationFailure(failure)).toEqual({ code, timedOut });
expect(failure).not.toHaveProperty("code");
expect(failure).not.toHaveProperty("cause");
expect(failure).not.toHaveProperty("stderr");
expect(JSON.stringify(failure)).toBe("{}");
expect((failure as Error).message).not.toContain("fixture-password");
expect(await stat(target).catch(() => null)).toBeNull();
});
it("writes the pinned receipt inside the clone before making it visible", async () => {
const { source, target } = await fixture();
const result = await materializeManagedProjectWorkspace(target, { repoUrl: "https://github.com/test/source", localSource: source, beforePublish: async clone => {
+1 -1
View File
@@ -8014,7 +8014,7 @@ export function issueRoutes(
"This issue is linked to a closed workspace that cannot be reopened.",
});
} else {
res.status(409).json({ error: result.message });
res.status(409).json({ error: result.message, ...(result.diagnostic ? { diagnostic: result.diagnostic } : {}) });
}
return null;
}
+37 -15
View File
@@ -20,7 +20,8 @@ import { hasPendingNestedCloneMigration, isNestedCloneMigrationCandidate, nextNe
NESTED_CLONE_MIGRATION_RETRY_REASONS, type NestedCloneMigrationRotation } from "./nested-clone-migration.js";
import type { StorageService } from "../storage/types.js";
import { getStorageService } from "../storage/index.js";
import { readGitCacheRestoration, restoreGitCacheBase, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, gitRestorePointExpired } from "./git-workspace-restoration.js";
import { readGitCacheRestoration, restoreGitCacheBase, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, gitRestorePointExpired,
classifyWorkspaceRestorationError, type GitCacheRestorationStep, type WorkspaceRestorationFailureKind } from "./git-workspace-restoration.js";
export const GIT_RESTORE_POINT_EXPIRED_WARNING = "The saved state of this workspace expired, so it was reopened as a fresh workspace on its branch. Unpushed and uncommitted work from before it was cleaned up is no longer available.";
import { parseObject } from "../adapters/utils.js";
@@ -327,7 +328,16 @@ export type ReopenClosedIsolatedExecutionWorkspaceResult =
// generation, so a stale actor never clears a newer reopen's fence.
| { ok: true; workspace: ExecutionWorkspace; reopened: true; generation: number; warning?: string }
| { ok: true; workspace: ExecutionWorkspace; reopened: false; generation: number }
| { ok: false; code: "not_reopenable" | "rebuild_failed"; message: string };
| { ok: false; code: "not_reopenable" | "rebuild_failed"; message: string; diagnostic?: WorkspaceReopenDiagnostic };
export type WorkspaceReopenFailureStep = GitCacheRestorationStep | "managed_base" | "copy_identity" | "expiration_locks"
| "workspace_rebuild" | "publish" | "transaction";
export type WorkspaceReopenDiagnostic = { step: WorkspaceReopenFailureStep; kind: WorkspaceRestorationFailureKind };
export function workspaceReopenDiagnostic(step: WorkspaceReopenFailureStep, error: unknown): WorkspaceReopenDiagnostic {
if (error instanceof GitCacheRestorationFailure && error.step) return { step: error.step, kind: classifyWorkspaceRestorationError(error) };
return { step, kind: error === undefined ? "unknown" : classifyWorkspaceRestorationError(error) };
}
export type ExecutionWorkspaceServiceOptions = {
pluginWorkerManager?: WorkspaceWorkers;
@@ -4345,9 +4355,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
});
}
await assertTaskWorkspaceExecutionAccess(db, input.authorizationActor, issue.companyId, { issueId: issue.id, workspaceId: snapshot.id });
let recipe: ReturnType<typeof readGitCacheRestoration>;
let recipe: ReturnType<typeof readGitCacheRestoration> = null;
const rebuildFailed = (step: WorkspaceReopenFailureStep, error: unknown, restoration = recipe !== null): ReopenClosedIsolatedExecutionWorkspaceResult => {
const diagnostic = workspaceReopenDiagnostic(step, error);
logger.warn({ event: "execution_workspace.reopen", outcome: "rebuild_failed", executionWorkspaceId: snapshot.id,
issueId: issue.id, companyId: issue.companyId, ...diagnostic }, "execution workspace reopen rebuild failed");
return { ok: false, code: "rebuild_failed", diagnostic,
message: restoration ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
};
try { recipe = readGitCacheRestoration(snapshot); }
catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; }
catch (error) { return rebuildFailed("recipe", error, true); }
const freshStart = recipe !== null && isClosedExecutionWorkspaceStatus(snapshot.status) && gitRestorePointExpired(recipe);
const gitAuth = createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id);
const [
@@ -4383,9 +4400,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
resolvedBaseCwd = recipe && isClosedExecutionWorkspaceStatus(snapshot.status)
? await restoreGitCacheBase(snapshot, recipe, gitAuth, undefined, { freshStart })
: await verifiedRecordedIsolatedRemoteSource(snapshot) ?? projectWorkspace?.cwd ?? null;
} catch {
return { ok: false, code: "rebuild_failed", message: recipe
? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
} catch (error) {
return rebuildFailed("source_verify", error);
}
if (isClosedExecutionWorkspaceStatus(snapshot.status) && resolvedBaseCwd == null && snapshot.strategyType === "git_worktree" && snapshot.projectId) {
try {
@@ -4409,8 +4425,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
resolveGitAuth: createIssueGitRemoteAuthProvider(db, snapshot.companyId, issue.id),
});
resolvedBaseCwd = managedWorkspace.cwd;
} catch {
return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
} catch (error) {
return rebuildFailed("managed_base", error);
}
}
let copyIdentity: { sourceCwd: string; configuredSourceCwd: string; targetCwd: string } | null = null;
@@ -4427,8 +4443,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
sourceCwd: await fs.realpath(sourceRoot),
configuredSourceCwd: await fs.realpath(resolvedBaseCwd), targetCwd,
};
} catch {
return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
} catch (error) {
return rebuildFailed("copy_identity", error);
}
}
const result = await db.transaction(async (tx): Promise<ReopenClosedIsolatedExecutionWorkspaceResult> => {
@@ -4478,7 +4494,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
}
try { await recoverGitCacheExpirationLocks(row); }
catch { return { ok: false, code: "rebuild_failed", message: new GitCacheRestorationFailure().message }; }
catch (error) { return rebuildFailed("expiration_locks", error, true); }
const verifyCopyIdentity = copyIdentity ? async (sourceCwd: string, targetCwd: string) => {
if (await fs.realpath(sourceCwd) !== copyIdentity.sourceCwd
|| await fs.realpath(resolvedBaseCwd!) !== copyIdentity.configuredSourceCwd
@@ -4525,6 +4541,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
});
let rebuildError: string | null = null;
let rebuildDiagnostic: WorkspaceReopenDiagnostic | null = null;
let taskDirectoryWarning: string | null = null;
const taskDirectoryState = readTaskDirectoryRetention(row);
try {
@@ -4644,9 +4661,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
});
if (!realized) {
rebuildError = "Execution workspace could not be rebuilt";
rebuildDiagnostic = workspaceReopenDiagnostic("workspace_rebuild", undefined);
}
} catch (error) {
rebuildError = error instanceof Error ? error.message : String(error);
rebuildDiagnostic = workspaceReopenDiagnostic("workspace_rebuild", error);
}
if (rebuildError) {
@@ -4677,6 +4696,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
actorType: actor.actorType,
actorAgentId: actor.agentId ?? null,
generation: nextGeneration,
...rebuildDiagnostic,
error: rebuildError,
},
"execution workspace reopen rebuild failed",
@@ -4693,9 +4713,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
issueId: issue.id,
outcome: "rebuild_failed",
generation: nextGeneration,
...rebuildDiagnostic,
},
});
return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
return { ok: false, code: "rebuild_failed", ...(rebuildDiagnostic ? { diagnostic: rebuildDiagnostic } : {}),
message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
}
// The rebuild succeeded. Publish the row as active in one write, and clear
@@ -4730,7 +4752,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
.returning()
.then((rows) => rows[0] ?? null);
if (!activeRow) {
return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
return rebuildFailed("publish", undefined);
}
logger.info(
{
@@ -4770,7 +4792,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
...(reopenWarning ? { warning: reopenWarning } : {}) };
}).catch((error): ReopenClosedIsolatedExecutionWorkspaceResult => {
logger.warn({ error, executionWorkspaceId: snapshot.id }, "Execution workspace reopen failed");
return { ok: false, code: "rebuild_failed", message: recipe ? new GitCacheRestorationFailure().message : "Failed to rebuild the execution workspace" };
return rebuildFailed("transaction", error);
});
return result;
};
+27 -21
View File
@@ -10,7 +10,7 @@ import type { GitRemoteAuthProvider } from "./git-credentials.js";
import { localRepositoryPath } from "./local-repository-path.js";
import { managedProjectCheckoutName } from "./managed-project-checkout-name.js";
import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js";
import { ensureSavedGitCommit, gitCacheCommand, GitCacheRestorationFailure } from "./git-workspace-restoration.js";
import { ensureSavedGitCommit, gitCacheCommand, GitCacheRestorationFailure, restorationFailureAt } from "./git-workspace-restoration.js";
import { applyGitRestorePointToWorktree, ensureGitRestorePointCommit, loadGitRestorePoint, type GitRestorePointRef } from "./git-restore-point-storage.js";
import { copyVerifiedGitExcludes } from "./git-repository-excludes.js";
import { importRestorePoint } from "./git-restore-point.js";
@@ -291,30 +291,36 @@ export async function restoreGitWorkspaceContents(input: Identity & { cwd: strin
}
const savedRestorePoint = checkout.restorePoint && input.isolateRepositories
? await loadGitRestorePoint({ storage: input.storage ?? getStorageService(), workspace, ref: checkout.restorePoint }) : null;
let preparationFailure: GitCacheRestorationFailure | null = null;
const result = await materializeManagedProjectWorkspace(destination, { repoUrl: checkout.source.repoUrl,
gitObjectSource: gitObjectSource ? { cwd: gitObjectSource, commits: [checkout.commit, ...(savedRestorePoint?.manifest.prerequisites ?? [])] } : undefined,
resolveGitAuth: input.resolveGitAuth, beforePublish: async temporary => {
if (gitObjectSource && checkout.restorePoint) await copyVerifiedGitExcludes(gitObjectSource, temporary);
if (savedRestorePoint) {
const { payload, manifest } = savedRestorePoint;
await ensureCheckoutPrerequisites(temporary, checkout, manifest, input.resolveGitAuth);
await importRestorePoint({ repositoryCwd: temporary, payload });
} else if (checkout.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage ?? getStorageService(), workspace,
ref: checkout.restorePoint, repositoryCwd: temporary });
await ensureSavedGitCommit(temporary, checkout.source.repoUrl, checkout.commit, input.resolveGitAuth);
if (checkout.branchName) await gitCacheCommand(temporary, ["check-ref-format", "--branch", checkout.branchName]);
await gitCacheCommand(temporary, checkout.branchName
? ["checkout", "-B", checkout.branchName, checkout.commit] : ["checkout", "--detach", checkout.commit]);
if (checkout.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage ?? getStorageService(), workspace,
ref: checkout.restorePoint, repositoryCwd: temporary, worktreePath: temporary, preserveImportedHistory: input.isolateRepositories });
await gitCacheCommand(temporary, ["reflog", "expire", "--expire=now", "--all"]);
await restoreSettings(temporary, checkout.settings, input, relative);
await input.authorizeSource(checkout.source);
await recheck?.();
if (gitObjectSource && !await isVerifiedNestedRepositoryBase(gitObjectSource, candidates!, checkout.source.repoUrl)) throw new GitCacheRestorationFailure();
} });
try {
if (gitObjectSource && checkout.restorePoint) await copyVerifiedGitExcludes(gitObjectSource, temporary);
if (savedRestorePoint) {
const { payload, manifest } = savedRestorePoint;
await ensureCheckoutPrerequisites(temporary, checkout, manifest, input.resolveGitAuth);
await importRestorePoint({ repositoryCwd: temporary, payload });
} else if (checkout.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage ?? getStorageService(), workspace,
ref: checkout.restorePoint, repositoryCwd: temporary });
await ensureSavedGitCommit(temporary, checkout.source.repoUrl, checkout.commit, input.resolveGitAuth);
if (checkout.branchName) await gitCacheCommand(temporary, ["check-ref-format", "--branch", checkout.branchName]);
await gitCacheCommand(temporary, checkout.branchName
? ["checkout", "-B", checkout.branchName, checkout.commit] : ["checkout", "--detach", checkout.commit]);
if (checkout.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage ?? getStorageService(), workspace,
ref: checkout.restorePoint, repositoryCwd: temporary, worktreePath: temporary, preserveImportedHistory: input.isolateRepositories });
await gitCacheCommand(temporary, ["reflog", "expire", "--expire=now", "--all"]);
await restoreSettings(temporary, checkout.settings, input, relative);
await input.authorizeSource(checkout.source);
await recheck?.();
if (gitObjectSource && !await isVerifiedNestedRepositoryBase(gitObjectSource, candidates!, checkout.source.repoUrl)) throw new GitCacheRestorationFailure();
} catch (error) {
preparationFailure = restorationFailureAt("workspace_contents", error);
throw preparationFailure;
}
} }).catch(error => { throw preparationFailure ?? error; });
if (result.warning) throw new GitCacheRestorationFailure();
}
await restoreSettings(input.cwd, input.contents.settings, input, CLAUDE_SETTINGS_PATH);
} catch { throw new GitCacheRestorationFailure(); }
} catch (error) { throw restorationFailureAt("workspace_contents", error); }
}
@@ -8,7 +8,8 @@ import path from "node:path";
import { execFile as execFileCallback } from "node:child_process";
import { createHash } from "node:crypto";
import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js";
import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js";
import { materializeManagedProjectWorkspace, RepositoryRefNotFoundError, readManagedRepositoryPreparationFailure } from "./managed-repository-checkout.js";
import { readGitConnectionFailure } from "./git-connection-failure.js";
import { promisify } from "node:util";
import type { GitRemoteAuthProvider } from "./git-credentials.js";
import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js";
@@ -85,8 +86,63 @@ export async function verifiedManagedGitObjectSource(workspace: Workspace, sourc
return sourcePath;
}
export type GitCacheRestorationStep = "recipe" | "trash_recovery" | "source_check" | "source_materialize" | "source_verify"
| "restore_point" | "saved_commit" | "workspace_contents";
export type WorkspaceRestorationFailureKind = "remote_unavailable" | "auth_rejected" | "commit_unavailable" | "timeout"
| "verification" | "filesystem" | "git_command" | "unknown";
export class GitCacheRestorationFailure extends Error {
constructor() { super("The saved workspace commit could not be restored. Check repository access and retry."); }
readonly step: GitCacheRestorationStep | null;
readonly kind: WorkspaceRestorationFailureKind | null;
constructor(diagnostic: { step: GitCacheRestorationStep; kind: WorkspaceRestorationFailureKind } | null = null) {
super("The saved workspace commit could not be restored. Check repository access and retry.");
this.step = diagnostic?.step ?? null;
this.kind = diagnostic?.kind ?? null;
}
}
export function classifyWorkspaceRestorationError(error: unknown): WorkspaceRestorationFailureKind {
if (error instanceof GitCacheRestorationFailure) return error.kind ?? "verification";
if (!(error instanceof Error)) return "unknown";
const connection = readGitConnectionFailure(error);
if (connection) {
if (connection.reason === "authentication_failed") return "auth_rejected";
return connection.reason === "connection_timeout" ? "timeout" : "remote_unavailable";
}
if (error instanceof RepositoryRefNotFoundError) return "commit_unavailable";
const details = error as Error & { code?: unknown; killed?: unknown; stderr?: unknown };
const preparationFailure = readManagedRepositoryPreparationFailure(error);
const commandCode = preparationFailure?.code ?? details.code;
const code = typeof commandCode === "string" ? commandCode : null;
if (commandCode === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") return "git_command";
if (preparationFailure?.timedOut || details.killed === true || commandCode === "ETIMEDOUT" || error.name === "AbortError" || error.name === "TimeoutError") return "timeout";
if (["ENOTFOUND", "EAI_AGAIN", "ECONNREFUSED", "ECONNRESET", "EHOSTUNREACH", "ENETUNREACH"].includes(code ?? "")) return "remote_unavailable";
if (["ENOENT", "ENOTDIR", "EISDIR", "EACCES", "EPERM", "ENOSPC", "EROFS", "EMFILE", "ENFILE", "ELOOP", "ENAMETOOLONG", "EBUSY", "EIO", "EDQUOT"].includes(code ?? "")) return "filesystem";
const stderr = typeof details.stderr === "string" ? details.stderr.slice(0, 16_384)
: Buffer.isBuffer(details.stderr) ? details.stderr.subarray(0, 16_384).toString("utf8") : error.message.slice(0, 16_384);
const lines = stderr.split(/\r?\n/).map(line => line.trim());
if (lines.some(line => /^(?:error|fatal): (?:cannot|could not|unable to) (?:open|create|write|lock|read|stat|mkdir|unlink|remove|rename)\b.*: (?:Permission denied|Operation not permitted|Read-only file system|No space left on device)\.?$/i.test(line)
|| /^(?:error|fatal): unable to access ['"](?![a-z][a-z0-9+.-]*:\/\/)[^'"\r\n]+['"]: (?:Permission denied|Operation not permitted|Read-only file system)\.?$/i.test(line))) return "filesystem";
if (lines.some(line => /^fatal: Authentication failed for ['"][^'"\r\n]+['"]$/i.test(line)
|| /^fatal: could not read (?:Username|Password) for ['"][^'"\r\n]+['"]: terminal prompts disabled$/i.test(line)
|| /^fatal: unable to access ['"][^'"\r\n]+['"]: The requested URL returned error: 40[13]$/i.test(line)
|| /^remote: (?:Invalid username or (?:password|token)|Permission [^\r\n]+ denied|Support for password authentication was removed|Password authentication is not supported)/i.test(line)
|| /^(?:[^\s:@]+@)?[a-z0-9][a-z0-9.-]*: Permission denied \((?:publickey|password|keyboard-interactive)(?:,[a-z-]+)*\)\.?$/i.test(line))) {
return "auth_rejected";
}
if (/not our ref|couldn't find remote ref|no such remote ref|unadvertised object|did not send all necessary objects/i.test(stderr)) {
return "commit_unavailable";
}
if (/could not resolve host|unable to access|could not read from remote repository|does not appear to be a git repository|repository .*not found|connection (refused|timed out|reset)/i.test(stderr)) {
return "remote_unavailable";
}
if (typeof details.stderr === "string" || typeof commandCode === "number") return "git_command";
return "unknown";
}
export function restorationFailureAt(step: GitCacheRestorationStep, error: unknown) {
if (error instanceof GitCacheRestorationFailure && error.step) return error;
return new GitCacheRestorationFailure({ step, kind: classifyWorkspaceRestorationError(error) });
}
export function readGitCacheRestoration(workspace: Workspace): GitCacheRestoration | null {
@@ -719,9 +775,12 @@ export function gitRestorePointExpired(recipe: GitCacheRestoration) {
export async function restoreGitCacheBase(workspace: Workspace, recipe: GitCacheRestoration, resolveGitAuth: GitRemoteAuthProvider,
storage?: StorageService, options: { freshStart?: boolean } = {}) {
if (readGitCacheRestoration(workspace)?.commit !== recipe.commit) throw new GitCacheRestorationFailure();
let step: GitCacheRestorationStep = "recipe";
try {
if (readGitCacheRestoration(workspace)?.commit !== recipe.commit) throw new GitCacheRestorationFailure();
step = "trash_recovery";
await finishGitCacheTrash(workspace);
step = "source_check";
const present = await fs.lstat(recipe.sourcePath).then(() => true, (error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return false;
throw error;
@@ -736,21 +795,25 @@ export async function restoreGitCacheBase(workspace: Workspace, recipe: GitCache
if (!entry.isDirectory() || entry.isSymbolicLink() || await fs.realpath(parent) !== parent) throw new GitCacheRestorationFailure();
}
}
step = present ? "source_verify" : "source_materialize";
const cwd = present ? await verifiedGitCacheSource(workspace, recipe.sourcePath)
: recipe.sourceKind === "isolated_remote"
? await materializeIsolatedRemoteWorkspaceSource({ companyId: recipe.companyId, issueId: recipe.sourceIssueId,
repoUrl: recipe.repoUrl, resolveGitAuth })
: (await materializeManagedProjectWorkspace(recipe.sourcePath, { repoUrl: recipe.repoUrl, resolveGitAuth })).cwd;
step = "source_verify";
await verifiedGitCacheSource(workspace, recipe.sourcePath);
if (!cwd) throw new GitCacheRestorationFailure();
if (options.freshStart) return cwd;
step = "restore_point";
if (recipe.restorePoint) {
await ensureGitRestorePointCommit({ storage: storage ?? getStorageService(), workspace: { companyId: workspace.companyId, id: workspace.id },
ref: recipe.restorePoint, repositoryCwd: cwd });
}
step = "saved_commit";
await ensureSavedGitCommit(cwd, recipe.repoUrl, recipe.commit, resolveGitAuth);
return cwd;
} catch { throw new GitCacheRestorationFailure(); }
} catch (error) { throw restorationFailureAt(step, error); }
}
export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit: string, resolveGitAuth: (url: string) => Promise<{ configArgs: string[]; env: Record<string, string> } | null>) {
@@ -762,7 +825,7 @@ export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit:
timeout: 60_000, maxBuffer: 1024 * 1024, env: { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0" },
});
if ((await gitCacheCommand(cwd, ["rev-parse", `${commit}^{commit}`])).trim() !== commit) throw new GitCacheRestorationFailure();
} catch { throw new GitCacheRestorationFailure(); }
} catch (error) { throw restorationFailureAt("saved_commit", error); }
}
/** Runtime callers must bind the recipe to the selected workspace and base. */
@@ -15,6 +15,29 @@ import { resolveRepositoryObjectCache, readRepositoryObjectCache, publishReposit
const execFile = promisify(execFileCallback);
const MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS = 10 * 60 * 1000;
type ManagedRepositoryPreparationFailure = { code: string | number | null; timedOut: boolean };
const preparationFailures = new WeakMap<Error, ManagedRepositoryPreparationFailure>();
export function readManagedRepositoryPreparationFailure(error: unknown): ManagedRepositoryPreparationFailure | null {
const failure = error instanceof Error ? preparationFailures.get(error) : undefined;
return failure ? { ...failure } : null;
}
function managedRepositoryPreparationError(message: string, error: unknown): Error {
const wrapped = new Error(message);
const retained = readManagedRepositoryPreparationFailure(error);
if (retained) preparationFailures.set(wrapped, retained);
else if (error instanceof Error) {
const details = error as Error & { code?: unknown; killed?: unknown };
const code = typeof details.code === "number" && Number.isSafeInteger(details.code) ? details.code
: typeof details.code === "string" && /^(?:E[A-Z0-9_]{1,63}|ABORT_ERR)$/.test(details.code) ? details.code : null;
const timedOut = code !== "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" && (details.killed === true || code === "ETIMEDOUT"
|| error.name === "AbortError" || error.name === "TimeoutError");
if (code !== null || timedOut) preparationFailures.set(wrapped, { code, timedOut });
}
return wrapped;
}
async function copyManagedGitObjects(cwd: string, repoUrl: string, source: { cwd: string; commits: readonly string[] }) {
const git = (directory: string, args: string[]) => execFile("git", ["-C", directory, ...args], {
env: { ...sanitizeRuntimeServiceBaseEnv(process.env), GIT_TERMINAL_PROMPT: "0" },
@@ -202,7 +225,7 @@ export async function materializeManagedProjectWorkspace(
if (error instanceof RepositoryRefNotFoundError) throw new RepositoryRefNotFoundError(message, error.requestedRef);
const connectionFailure = readGitConnectionFailure(error);
if (connectionFailure) throw new GitConnectionFailureError(message, connectionFailure);
throw new Error(message);
throw managedRepositoryPreparationError(message, error);
}
try {
@@ -216,8 +239,9 @@ export async function materializeManagedProjectWorkspace(
}
const reason =
renameError instanceof Error ? renameError.message : String(renameError);
throw new Error(
throw managedRepositoryPreparationError(
`Failed to move managed checkout into place at "${cwd}": ${reason}`,
renameError,
);
}
return { cwd, warning: null };