diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0e4de36e84..157589b00c 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -632,6 +632,8 @@ When effective run config changes, Paperclip may intentionally skip a saved adap Paperclip applies one process-wide scheduler to expensive host-side workspace Git enumeration, including changed-file browsing, runtime/finalization cleanliness guards, and adapter sandbox-sync snapshots. The scheduler defaults to two active scans and a bounded queue of 32. Identical scans of the same canonical worktree share one subprocess, while successful changed-file listings are cached for 10 seconds. Correctness-sensitive runtime guards bypass the result cache. +Workspace snapshots list ignored paths with `git ls-files --others --ignored --exclude-standard --directory -z` so ignored directory contents do not require a full status walk. Snapshot failures retain their typed cause instead of becoming a non-Git-folder result. During pre-provider setup, scan timeouts and queue saturation use the existing two automatic failure retries with a 30-second delay. Cancellation, output limits, and other Git errors stop with specific recovery guidance. See `doc/execution-semantics.md` for the ownership and retry-budget contract. + The cache intentionally trades up to a few seconds of changed-file freshness for stable server latency. The file browser retains an explicit refresh action, does not start its query while the panel or browser tab is hidden, and presents overloads as retryable failures rather than an empty workspace. A full queue returns `503` with code `workspace_git_scan_saturated`; a scan exceeding its wall-clock limit returns `504` with code `workspace_git_scan_timeout`. Both responses include `Retry-After: 1`. Sandbox Git sync treats only the selected repository root as a clone source. A selected subfolder uses directory sync within that folder, applies the enclosing repository's ignore rules, and does not transfer parent files or Git history. diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 3bdaadac23..1df3a5f623 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -411,6 +411,26 @@ Workspace incoherence feeds into the same non-terminal liveness and stranded ass For runtime-created `git_worktree` execution workspaces, branch coherence is part of workspace coherence. The persisted execution workspace branch is the recorded branch for future dispatch. Reusing that workspace must verify that the worktree is still registered and that `HEAD` is on the recorded branch. Successful run finalization must perform the same check before recording `workspace_finalize=succeeded`. If the run switched to a publishing/PR branch without updating the execution workspace record, finalization may auto-restore the recorded branch only when the worktree is clean, still registered, and the recorded branch points at the current `HEAD`; the repair is recorded as a workspace operation before the successful finalize row. If that safe repair cannot be proven, finalization records a failed workspace finalize and the run fails with bounded evidence for the expected and actual branch. A branch change is sanctioned when a control-plane path updates the execution workspace record before finalization, when publishing work happens in a separate worktree and the managed issue worktree remains on its recorded branch, or when the finalizer performs this clean same-commit restoration. +### Workspace scan failures before provider startup + +Repository discovery distinguishes an ordinary folder from a failed Git read. +A timeout, full scan queue, cancellation, output limit, or Git failure must keep +its typed cause through workspace preparation and run persistence. It must not +be reported as a missing repository or fall back to an unfiltered directory copy. + +When workspace preparation fails before provider work starts, scan timeouts and +queue saturation use the existing durable failure budget: two automatic retries, +30 seconds apart. The scheduled successor is persisted before execution is +released. Restart and duplicate wake handling reuse that successor. Normal task, +ownership, pause, dependency, approval, and budget gates still apply. Existing +workspace content is retained, and incomplete temporary clones are not published. + +Cancelled scans, output-limit failures, and other Git failures do not authorize +an automatic setup retry. Exhaustion or an unsafe retry opens the source-scoped +recovery path with the specific scan cause and an operator action. Generic +stranded-work recovery must not grant another budget for these errors. This +does not automatically replay historical generic `setup_failed` runs. + ### ACP startup handshake bound An adapter-backed live path also requires that the ACP startup handshake itself cannot hang forever. The engine bounds the handshake with a fixed startup deadline and a poll of the duplex control-channel disposition. Either condition ends the handshake and reports a closed, typed code, so the issue can reach a settled disposition instead of staying `in_progress` with no observable next action. diff --git a/packages/adapter-utils/src/git-workspace-sync.test.ts b/packages/adapter-utils/src/git-workspace-sync.test.ts index 4d94ecf45b..163df791b3 100644 --- a/packages/adapter-utils/src/git-workspace-sync.test.ts +++ b/packages/adapter-utils/src/git-workspace-sync.test.ts @@ -105,6 +105,35 @@ describe("git workspace sync", () => { expect(snapshot?.ignoredPaths).toContain(ignoredName); }); + it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "workspace_git_scan_failed"])("preserves %s instead of reporting a non-Git folder", async (code) => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-scan-failure-")); + cleanupDirs.push(rootDir); + const repo = await createRepo(rootDir); + const failure = Object.assign(new Error("Git enumeration failed"), { code }); + setExpensiveWorkspaceGitExecutor(async (input) => { + if (input.operation === "adapter_sync.ignored_files") throw failure; + return runLocalGit(input.localDir, [...input.args]); + }); + await expect(readGitWorkspaceSnapshot(repo, false)).rejects.toBe(failure); + }); + + it("lists ignored paths without traversing ignored directory contents", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-ignored-scan-")); + cleanupDirs.push(rootDir); + const repo = await createRepo(rootDir); + await writeFile(path.join(repo, ".gitignore"), "dependencies/\n*.secret\n"); + await mkdir(path.join(repo, "dependencies", "nested"), { recursive: true }); + await writeFile(path.join(repo, "dependencies", "nested", "private"), "private"); + await writeFile(path.join(repo, "token.secret"), "private"); + let ignoredArgs: readonly string[] = []; + setExpensiveWorkspaceGitExecutor(async (input) => { + if (input.operation === "adapter_sync.ignored_files") ignoredArgs = input.args; + return runLocalGit(input.localDir, [...input.args]); + }); + expect((await readGitWorkspaceSnapshot(repo))?.ignoredPaths).toEqual(["dependencies", "token.secret"]); + expect(ignoredArgs).toEqual(["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"]); + }); + async function createRepo(rootDir: string): Promise { const repo = path.join(rootDir, "repo"); await mkdir(repo, { recursive: true }); diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index 3ee7450052..42bc21250e 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -159,87 +159,91 @@ export async function readGitWorkspaceSnapshot(localDir: string, includeReposito } } } + // Only repository discovery may report an ordinary directory. A failed + // snapshot of a confirmed repository must never fall back to directory sync. + let insideWorkTree: GitCommandResult; try { - const insideWorkTree = await runLocalGit(localDir, ["rev-parse", "--is-inside-work-tree"], { + insideWorkTree = await runLocalGit(localDir, ["rev-parse", "--is-inside-work-tree"], { timeout: 10_000, maxBuffer: 16 * 1024, }); - if (insideWorkTree.stdout.trim() !== "true") { - return null; - } - - const toplevelResult = await runLocalGit(localDir, ["rev-parse", "--show-toplevel"], { - timeout: 10_000, - maxBuffer: 16 * 1024, - }); - // Git discovers a parent repository from a nested project directory, but - // that directory is not a fetch source. Keep the selected workspace - // boundary: subfolders use directory sync instead of importing the parent. - const [workspacePath, repositoryPath] = await Promise.all([ - fs.realpath(localDir), - fs.realpath(toplevelResult.stdout.trim()), - ]); - if (workspacePath !== repositoryPath) return null; - - const [headCommitResult, branchResult, overlayDiffResult, untrackedResult, deletedResult, ignoredResult] = await Promise.all([ - runLocalGit(localDir, ["rev-parse", "HEAD"], { - timeout: 10_000, - maxBuffer: 16 * 1024, - }), - runLocalGit(localDir, ["rev-parse", "--abbrev-ref", "HEAD"], { - timeout: 10_000, - maxBuffer: 16 * 1024, - }), - runExpensiveWorkspaceGit(localDir, ["diff", "--name-only", "-z", "--diff-filter=ACMRTUXB", "HEAD", "--"], "adapter_sync.overlay_diff", { - timeout: 10_000, - maxBuffer: 1024 * 1024, - }), - runExpensiveWorkspaceGit(localDir, ["ls-files", "--others", "--exclude-standard", "-z"], "adapter_sync.untracked_files", { - timeout: 10_000, - maxBuffer: 1024 * 1024, - }), - runExpensiveWorkspaceGit(localDir, ["diff", "--name-only", "-z", "--diff-filter=D", "HEAD", "--"], "adapter_sync.deleted_files", { - timeout: 10_000, - maxBuffer: 256 * 1024, - }), - runExpensiveWorkspaceGit(localDir, ["status", "--ignored", "--porcelain=v1", "-z", "--untracked-files=normal"], "adapter_sync.ignored_files", { - timeout: 10_000, - maxBuffer: 1024 * 1024, - }), - ]); - - const branchName = branchResult.stdout.trim(); - // `-z` already delimits each record with a NUL byte, so a leading or - // trailing space in a record is part of the path itself, not padding to - // remove — trimming it would resolve to a path that does not exist. A - // length check finds the one genuinely empty record `-z` appends after - // the last NUL, without eating a real path's own leading or trailing - // whitespace. This applies to all four NUL-delimited outputs below (the - // overlay diff, the untracked list, the deleted list, and the ignored - // list); `branchName` and `headCommit` come from non-`-z` commands and - // keep their own `.trim()` above and below, which is safe. - const splitNul = (value: string) => value.split("\0").filter((entry) => entry.length > 0); - return { - headCommit: headCommitResult.stdout.trim(), - branchName: branchName && branchName !== "HEAD" ? branchName : null, - overlayPaths: [...new Set([...splitNul(overlayDiffResult.stdout), ...splitNul(untrackedResult.stdout), - ...repositories.flatMap((repo) => repo.snapshot.overlayPaths.map((entry) => `${repo.path}/${entry}`))])] - .sort((left, right) => left.localeCompare(right)), - deletedPaths: [...new Set([...splitNul(deletedResult.stdout), - ...repositories.flatMap((repo) => repo.snapshot.deletedPaths.map((entry) => `${repo.path}/${entry}`))])] - .sort((left, right) => left.localeCompare(right)), - ignoredPaths: [...splitNul(ignoredResult.stdout) - .filter((entry) => entry.startsWith("!! ")) - .map((entry) => entry.slice(3).replace(/\/+$/, "")) - .filter((entry) => Boolean(entry) && !(repositories.length > 0 && entry === PROJECT_REPOSITORIES_DIR)), - ...repositories.flatMap((repo) => repo.snapshot.ignoredPaths.map((entry) => `${repo.path}/${entry}`))] - .sort((left, right) => left.localeCompare(right)), - ...(repositories.length > 0 ? { repositories } : {}), - }; } catch (error) { - if (repositories.length > 0) throw error; + if (repositories.length === 0 && isNotAGitRepositoryError(error)) return null; + throw error; + } + if (insideWorkTree.stdout.trim() !== "true") { return null; } + + const toplevelResult = await runLocalGit(localDir, ["rev-parse", "--show-toplevel"], { + timeout: 10_000, + maxBuffer: 16 * 1024, + }); + // Git discovers a parent repository from a nested project directory, but + // that directory is not a fetch source. Keep the selected workspace + // boundary: subfolders use directory sync instead of importing the parent. + const [workspacePath, repositoryPath] = await Promise.all([ + fs.realpath(localDir), + fs.realpath(toplevelResult.stdout.trim()), + ]); + if (workspacePath !== repositoryPath) return null; + + const [headCommitResult, branchResult, overlayDiffResult, untrackedResult, deletedResult, ignoredResult] = await Promise.all([ + runLocalGit(localDir, ["rev-parse", "HEAD"], { + timeout: 10_000, + maxBuffer: 16 * 1024, + }), + runLocalGit(localDir, ["rev-parse", "--abbrev-ref", "HEAD"], { + timeout: 10_000, + maxBuffer: 16 * 1024, + }), + runExpensiveWorkspaceGit(localDir, ["diff", "--name-only", "-z", "--diff-filter=ACMRTUXB", "HEAD", "--"], "adapter_sync.overlay_diff", { + timeout: 10_000, + maxBuffer: 1024 * 1024, + }), + runExpensiveWorkspaceGit(localDir, ["ls-files", "--others", "--exclude-standard", "-z"], "adapter_sync.untracked_files", { + timeout: 10_000, + maxBuffer: 1024 * 1024, + }), + runExpensiveWorkspaceGit(localDir, ["diff", "--name-only", "-z", "--diff-filter=D", "HEAD", "--"], "adapter_sync.deleted_files", { + timeout: 10_000, + maxBuffer: 256 * 1024, + }), + // Collapse ignored directories instead of walking their contents, and + // avoid producing unrelated tracked/untracked status records. + runExpensiveWorkspaceGit(localDir, ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"], "adapter_sync.ignored_files", { + timeout: 10_000, + maxBuffer: 1024 * 1024, + }), + ]); + + const branchName = branchResult.stdout.trim(); + // `-z` already delimits each record with a NUL byte, so a leading or + // trailing space in a record is part of the path itself, not padding to + // remove — trimming it would resolve to a path that does not exist. A + // length check finds the one genuinely empty record `-z` appends after + // the last NUL, without eating a real path's own leading or trailing + // whitespace. This applies to all four NUL-delimited outputs below (the + // overlay diff, the untracked list, the deleted list, and the ignored + // list); `branchName` and `headCommit` come from non-`-z` commands and + // keep their own `.trim()` above and below, which is safe. + const splitNul = (value: string) => value.split("\0").filter((entry) => entry.length > 0); + return { + headCommit: headCommitResult.stdout.trim(), + branchName: branchName && branchName !== "HEAD" ? branchName : null, + overlayPaths: [...new Set([...splitNul(overlayDiffResult.stdout), ...splitNul(untrackedResult.stdout), + ...repositories.flatMap((repo) => repo.snapshot.overlayPaths.map((entry) => `${repo.path}/${entry}`))])] + .sort((left, right) => left.localeCompare(right)), + deletedPaths: [...new Set([...splitNul(deletedResult.stdout), + ...repositories.flatMap((repo) => repo.snapshot.deletedPaths.map((entry) => `${repo.path}/${entry}`))])] + .sort((left, right) => left.localeCompare(right)), + ignoredPaths: [...splitNul(ignoredResult.stdout) + .map((entry) => entry.replace(/\/+$/, "")) + .filter((entry) => Boolean(entry) && !(repositories.length > 0 && entry === PROJECT_REPOSITORIES_DIR)), + ...repositories.flatMap((repo) => repo.snapshot.ignoredPaths.map((entry) => `${repo.path}/${entry}`))] + .sort((left, right) => left.localeCompare(right)), + ...(repositories.length > 0 ? { repositories } : {}), + }; } /** The `git ls-files --others --ignored` output for one directory, read by {@link readReferencedSourceGitIgnoredPaths}. */ diff --git a/server/src/__tests__/heartbeat-project-repositories.test.ts b/server/src/__tests__/heartbeat-project-repositories.test.ts index cd618f0116..932c093d36 100644 --- a/server/src/__tests__/heartbeat-project-repositories.test.ts +++ b/server/src/__tests__/heartbeat-project-repositories.test.ts @@ -1,12 +1,14 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { execFileSync } from "node:child_process"; import { mkdtemp, mkdir, writeFile, readFile, realpath, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, createDb, issues, projects, projectWorkspaces } from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; +import { runLocalGit, setExpensiveWorkspaceGitExecutor } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { WorkspaceGitScanError } from "../services/workspace-git-operation-scheduler.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.ts"; import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; @@ -44,6 +46,103 @@ suite("task project repository provisioning", () => { vi.unstubAllEnvs(); await rm(root, { recursive: true, force: true }); }, 60_000); + afterEach(async () => { + await drainHeartbeatRunsToQuiescence(db, heartbeat); + setExpensiveWorkspaceGitExecutor(null); + }); + + it.each([ + { code: "workspace_git_scan_timeout", scenario: "temporary", retryable: true }, + { code: "workspace_git_scan_saturated", scenario: "temporary", retryable: true }, + { code: "workspace_git_scan_timeout", scenario: "exhausted", retryable: true }, + { code: "workspace_git_scan_timeout", scenario: "paused", retryable: true }, + { code: "workspace_git_scan_output_limit", scenario: "permanent", retryable: false }, + { code: "workspace_git_scan_cancelled", scenario: "cancelled", retryable: false }, + { code: "workspace_git_scan_failed", scenario: "permanent", retryable: false }, + ] as const)("handles $scenario $code during local-source preparation", async ({ code, scenario, retryable }) => { + const companyId = randomUUID(), projectId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + const source = path.join(root, companyId, "source"); + await mkdir(source, { recursive: true }); + const git = (...args: string[]) => execFileSync("git", args, { cwd: source, stdio: "ignore" }); + git("init", "-b", "main"); + await writeFile(path.join(source, "README.md"), "committed"); + await writeFile(path.join(source, ".gitignore"), "private.secret\n"); + git("add", "."); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "seed"); + await writeFile(path.join(source, "README.md"), "preserved dirty work"); + await writeFile(path.join(source, "private.secret"), "must not copy"); + await db.insert(companies).values({ id: companyId, name: "Bootstrap recovery", issuePrefix: `R${companyId.slice(0, 6)}`, defaultResponsibleUserId: "responsible-user" }); + await db.insert(projects).values({ id: projectId, companyId, name: "Local source", status: "in_progress" }); + await db.insert(projectWorkspaces).values([ + { id: randomUUID(), companyId, projectId, name: "Anchor", sourceType: "local_path", cwd: source, isPrimary: true, createdAt: new Date(Date.now() - 1000) }, + { id: randomUUID(), companyId, projectId, name: "Source copy", sourceType: "git_repo", repoUrl: pathToFileURL(source).href, cwd: source, isPrimary: false }, + ]); + await db.insert(agents).values({ id: agentId, companyId, name: "Test", role: "engineer", status: "idle", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, projectId, title: "Recover startup and use existing work", status: "todo", assigneeAgentId: agentId }); + let inject = true; + const canonicalSource = await realpath(source); + setExpensiveWorkspaceGitExecutor(async (input) => { + if (inject && (input.localDir === source || input.localDir === canonicalSource) && input.operation === "adapter_sync.ignored_files") { + inject = scenario === "exhausted"; + throw new WorkspaceGitScanError(code, "Injected temporary scan failure"); + } + return runLocalGit(input.localDir, [...input.args]); + }); + const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId, projectId } }); + await vi.waitFor(async () => expect((await heartbeat.getRun(run!.id))?.errorCode).toBe(code), { timeout: 15_000 }); + expect(execute.mock.calls.filter(([input]) => input.runId === run!.id)).toHaveLength(0); + await heartbeat.drainActiveRunExecutions(); + if (!retryable) { + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]?.status).toBe("blocked"); + return; + } + let retry: typeof heartbeatRuns.$inferSelect; + await vi.waitFor(async () => { + const rows = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id)); + expect(rows).toHaveLength(1); + retry = rows[0]!; + expect(retry.status).toBe("scheduled_retry"); + expect(retry.scheduledRetryAttempt).toBe(1); + }); + // Recreate the service as on restart; durable scheduling must not need the old closure. + await heartbeat.drainActiveRunExecutions(); + heartbeat = heartbeatService(db); + if (scenario === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(Date.now() - 1000) }).where(eq(heartbeatRuns.id, retry!.id)); + await heartbeat.promoteDueScheduledRetries(); + await heartbeat.promoteDueScheduledRetries(); + await heartbeat.resumeQueuedRuns(); + if (scenario === "paused") { + expect((await heartbeat.getRun(retry!.id))?.status).toBe("cancelled"); + expect(execute.mock.calls.filter(([input]) => input.agent.id === agentId)).toHaveLength(0); + return; + } + if (scenario === "exhausted") { + await heartbeat.drainActiveRunExecutions(); + expect((await heartbeat.getRun(retry!.id))?.errorCode).toBe(code); + const [last] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, retry!.id)); + expect(last).toMatchObject({ status: "scheduled_retry", scheduledRetryAttempt: 2 }); + await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(Date.now() - 1000) }).where(eq(heartbeatRuns.id, last!.id)); + await heartbeat.promoteDueScheduledRetries(); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + expect((await heartbeat.getRun(last!.id))?.errorCode).toBe(code); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))).toHaveLength(3); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", assigneeAgentId: agentId }); + expect(execute.mock.calls.filter(([input]) => input.agent.id === agentId)).toHaveLength(0); + return; + } + await vi.waitFor(async () => expect((await heartbeat.getRun(retry!.id))?.status).toBe("succeeded"), { timeout: 15_000 }); + expect(execute.mock.calls.filter(([input]) => input.runId === retry!.id)).toHaveLength(1); + const [task] = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))); + expect(task).toMatchObject({ status: "done", assigneeAgentId: agentId }); + const copies = (execute.mock.calls.find(([input]) => input.runId === retry!.id)![0].context.paperclipWorkspaces as Array<{ cwd: string }>).filter((hint) => hint.cwd.includes(".paperclip-repositories")); + expect(copies.length).toBeGreaterThan(0); + expect(await readFile(path.join(copies[0]!.cwd, "README.md"), "utf8")).toBe("preserved dirty work"); + await expect(readFile(path.join(copies[0]!.cwd, "private.secret"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(path.join(source, "README.md"), "utf8")).toBe("preserved dirty work"); + }, 40_000); it.each([1, 2])("gives a task all %i repositories without any configured local folders", async (count) => { const companyId = randomUUID(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 58cab31a15..27a32e95b2 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -29,6 +29,7 @@ export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status- import { buildExecutionContinuation } from "./execution-continuation.js"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; import { PROJECT_REPOSITORIES_DIR, readGitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { isWorkspaceGitScanError, WorkspaceGitScanError, WORKSPACE_GIT_SCAN_ERROR_CODES } from "./workspace-git-operation-scheduler.js"; import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { initializeRunIdentity, explicitOperatorRunIdentity } from "./run-identity.js"; import { @@ -761,6 +762,9 @@ export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; +function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { + return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; +} const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; export { @@ -2496,11 +2500,13 @@ async function materializeManagedProjectWorkspace( error: reason, used: auth ? { source: auth.source, secretName: auth.secretName } : null, }); - throw new Error( - scrubGitCredentialText( - `Failed to prepare managed checkout for "${input.repoUrl}" at "${cwd}": ${reason}${authNote ? ` ${authNote}` : ""}`, - ), + const message = scrubGitCredentialText( + `Failed to prepare managed checkout for "${input.repoUrl}" at "${cwd}": ${reason}${authNote ? ` ${authNote}` : ""}`, ); + // Preserve the closed failure code without copying subprocess output or + // credentials into the durable run. Setup recovery needs the actual cause. + if (isWorkspaceGitScanError(error)) throw new WorkspaceGitScanError(error.code, message); + throw new Error(message); } try { @@ -15243,6 +15249,7 @@ export function heartbeatService( : baseSchedule; const requiresIssueGate = + isTransientWorkspaceGitScanCode(run.errorCode) || hasConversationContinuationPolicy(run.resultJson) || retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || @@ -25282,7 +25289,9 @@ export function heartbeatService( ); const nonRetryablePreflightCode = nonRetryablePreflightFailureCode(outerErr); + const workspaceGitScanFailure = isWorkspaceGitScanError(outerErr) ? outerErr : null; const setupFailureErrorCode = + workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? (unresolvedBaseRefSetupFailure || @@ -25301,6 +25310,13 @@ export function heartbeatService( // action, so it is persisted even when the agent lookup failed and the // agent-scoped stop metadata cannot be merged in. const setupFailureDetails = + (workspaceGitScanFailure ? { + workspaceGitScan: { + code: workspaceGitScanFailure.code, + phase: "workspace_setup", + retryable: isTransientWorkspaceGitScanCode(workspaceGitScanFailure.code), + }, + } : null) ?? workspaceValidationSetupFailure?.resultJson ?? configurationIncompleteSetupFailure?.resultJson ?? (unresolvedBaseRefSetupFailure @@ -25403,9 +25419,12 @@ export function heartbeatService( () => undefined, ); } - await scheduleInteractionContinuationInfrastructureRetryIfEligible( - livenessRun, - failedAgent, + // No provider work began. Retry temporary host scan failures with + // the existing durable failure budget, before releasing execution. + // Generic recovery must not grant a second budget on exhaustion. + await (isTransientWorkspaceGitScanCode(livenessRun.errorCode) + ? scheduleBoundedRetryForRun(livenessRun, failedAgent) + : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent) ).catch((retryError) => { logger.warn( { err: retryError, runId: livenessRun.id }, diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.ts index e1b81f3cca..6df61bf3d0 100644 --- a/server/src/services/legacy-execution-recovery.test.ts +++ b/server/src/services/legacy-execution-recovery.test.ts @@ -9,6 +9,17 @@ const stopped = { }, }; +it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { + const run = { runtimeMode: "legacy", status: "failed", errorCode, scheduledRetryAttempt: 2, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; + expect(legacyExecutionNeedsReconciliation(run)).toBe(false); + expect(legacyExecutionNeedsReconciliation({ ...run, resultJson: {} })).toBe(true); + expect(legacyExecutionNeedsReconciliation({ ...run, resultJson: { + executionRecovery: { kind: "bootstrap", providerWorkStarted: true }, + } })).toBe(true); + expect(legacyExecutionNeedsReconciliation({ ...run, errorCode: "setup_failed" })).toBe(true); +}); + it("permits subscription waits only with explicit evidence that provider work never started", () => { const waiting = { runtimeMode: "legacy", status: "cancelled", errorCode: "ai_connection_busy", scheduledRetryAttempt: 12, diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index fa967dc937..1cdbeea720 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -37,6 +37,11 @@ export function legacyExecutionNeedsReconciliation( evidence?.kind === "ai_connection_wait" && evidence.providerWorkStarted === false) return false; if (run.status === "cancelled" && run.errorCode === "workspace_busy" && evidence?.kind === "workspace_wait" && evidence.providerWorkStarted === false) return false; + // Setup owns the bounded retry budget for temporary workspace scans. Its + // exhaustion needs workspace repair, not reconciliation of provider actions + // that the bootstrap evidence proves never started. Keep unknown outcomes held. + if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated") && + evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts index cd873a16a1..a3ad7a83b5 100644 --- a/server/src/services/recovery/service.pause-durability.test.ts +++ b/server/src/services/recovery/service.pause-durability.test.ts @@ -5,6 +5,9 @@ const run = (errorCode: string | null) => ({ errorCode } as unknown as Parameters[0]); describe("pause durability: continuation retry classification", () => { + it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled"])("does not grant %s another recovery budget", (code) => { + expect(classifyContinuationFailure(run(code))).toMatchObject({ kind: "non_retryable", maxAttempts: 0 }); + }); it("agent_paused is retryable so work resumes (Option A: Resume Continues Work)", () => { // Pause still emits errorCode agent_paused for observability, but it is NOT // non-retryable. On resume the agent becomes invokable again and this classifies diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 0929a346d1..bbbff367c6 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -489,6 +489,13 @@ const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set([ // process starts. Known transient preflight failures use dedicated bounded // retry paths instead of generic issue continuation recovery. "setup_failed", + // Setup owns the shared durable retry budget for temporary Git scans. + // Generic continuation must not retry permanent failures or reset that budget. + "workspace_git_scan_timeout", + "workspace_git_scan_saturated", + "workspace_git_scan_cancelled", + "workspace_git_scan_output_limit", + "workspace_git_scan_failed", "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", "low_trust_boundary_mismatch", diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index b561593e72..a4b838a744 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -76,6 +76,18 @@ describe("stranded recovery notice seeds", () => { }); describe("buildStrandedRecoveryEscalationNotice", () => { + it("names a workspace timeout and its repair instead of claiming generic continuation", () => { + const notice = buildStrandedRecoveryEscalationNotice({ + seed: buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), + recoveryActionId: "scan-recovery", + recoveryOwner: null, + sourceRun: { id: "failed-run", status: "failed", errorCode: "workspace_git_scan_timeout" }, + }); + expect(notice.presentation.title).toBe("Workspace scan timed out"); + expect(notice.body).toContain("before the agent started"); + expect(notice.body).not.toContain("retried continuation"); + expect(JSON.stringify(notice.metadata)).toContain("repository access and server load"); + }); const actionId = "6a2f8e64-6f5e-4b58-b7fd-111111111111"; const owner = { id: "9b1c2d3e-4f50-4a61-8b72-222222222222", name: "CTO" }; const sourceRun = { diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index acd1f64243..8d98ad3d8a 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -47,6 +47,14 @@ const STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE: Record acpx_auth_required: "Error: agent login required", }; +const WORKSPACE_SCAN_NOTICES: Record = { + workspace_git_scan_timeout: { title: "Workspace scan timed out", nextAction: "Check repository access and server load, then retry the task." }, + workspace_git_scan_saturated: { title: "Workspace scan queue is full", nextAction: "Check server load and the workspace scan queue, then retry the task." }, + workspace_git_scan_output_limit: { title: "Workspace scan exceeded its limit", nextAction: "Check the repository size and workspace scan output limit before retrying the task." }, + workspace_git_scan_failed: { title: "Workspace scan failed", nextAction: "Inspect the failed run and check repository access and integrity before retrying the task." }, + workspace_git_scan_cancelled: { title: "Workspace scan was cancelled", nextAction: "Inspect why workspace preparation was cancelled before retrying the task." }, +}; + export function buildImmediateExecutionPathRecoveryNoticeSeed(input: { status: "todo" | "in_progress"; }): StrandedRecoveryNoticeSeed { @@ -175,11 +183,17 @@ export function buildStrandedRecoveryEscalationNotice(input: { errorSummary?: string | null; } | null | undefined; }): StrandedRecoveryEscalationNotice { + const workspaceScan = WORKSPACE_SCAN_NOTICES[input.sourceRun?.errorCode ?? ""]; + const seed = workspaceScan ? { + ...workspaceScan, + body: `Paperclip could not prepare the workspace before the agent started. Automatic recovery could not continue. ${workspaceScan.nextAction}`, + tone: "danger" as const, + } : input.seed; const fallbackBody = input.fallbackBody?.trim(); - const body = input.seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY); + const body = seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY); const title = STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE[input.sourceRun?.errorCode?.trim() ?? ""] ?? - input.seed?.title ?? + seed?.title ?? STRANDED_RECOVERY_NOTICE_TITLES_BY_CAUSE[input.recoveryCause ?? ""] ?? DEFAULT_STRANDED_RECOVERY_NOTICE_TITLE; @@ -193,7 +207,7 @@ export function buildStrandedRecoveryEscalationNotice(input: { ), keyValueRow( "Next action", - input.seed?.nextAction ?? (input.recoveryOwner + seed?.nextAction ?? (input.recoveryOwner ? "The recovery owner should either restore a live execution path or record the manual resolution on the source issue" : "Inspect the evidence, then retry the original owner, explicitly reassign, repair the execution path, or record an intentional resolution"), ), @@ -215,7 +229,7 @@ export function buildStrandedRecoveryEscalationNotice(input: { return { body, - presentation: systemNoticePresentation({ tone: input.seed?.tone ?? "danger", title }), + presentation: systemNoticePresentation({ tone: seed?.tone ?? "danger", title }), metadata: { version: 1, sourceRunId: input.sourceRun?.id ?? null, diff --git a/tests/e2e/workspace-bootstrap/README.md b/tests/e2e/workspace-bootstrap/README.md new file mode 100644 index 0000000000..29f9458a8b --- /dev/null +++ b/tests/e2e/workspace-bootstrap/README.md @@ -0,0 +1,36 @@ +# Workspace bootstrap recovery acceptance + +This opt-in suite uses a disposable loopback instance, real Git repositories, +the real Git scan scheduler, durable retries, and the task UI. A deterministic +process adapter verifies the prepared files and completes its task through the +run-scoped API. It makes no model calls. The test Git executable injects a real +timeout only when the test explicitly arms a marker inside its repository. + +From a development worktree, start the foreground instance: + +```sh +BOOTSTRAP_FIXTURE_KEY=not-a-provider-key \ +PATH="$PWD/tests/e2e/workspace-bootstrap/bin:$PATH" \ +NODE_ENV=test \ +node cli/node_modules/tsx/dist/cli.mjs cli/src/index.ts test-drive \ + --harness codex --api-key-env BOOTSTRAP_FIXTURE_KEY \ + --company-name 'Workspace Recovery QA' --no-browser +``` + +Set `BOOTSTRAP_REAL_GIT` to the absolute Git executable if it is not +`/usr/bin/git`. Do not point this fixture at a normal development or production +instance. The seeder checks the loopback host, trusted-local mode, and company +name. It does not create tasks; each test creates its task through the UI. + +Use the ready URL printed by test-drive: + +```sh +WORKSPACE_BOOTSTRAP_TEST_URL=http://127.0.0.1:3100 \ +pnpm exec playwright test --config tests/e2e/workspace-bootstrap/playwright.config.ts +``` + +The tests prove that one timeout schedules a retry and completes without manual +Retry, that persistent timeouts stop after three total runs, and that the final +state survives reload. Source edits remain intact and ignored private files are +not copied. Screenshots and failure traces go to `test-results/workspace-bootstrap`. +Stop test-drive with Ctrl-C. Its disposable data directory remains for inspection. diff --git a/tests/e2e/workspace-bootstrap/bin/git b/tests/e2e/workspace-bootstrap/bin/git new file mode 100755 index 0000000000..92c7d9e7f6 --- /dev/null +++ b/tests/e2e/workspace-bootstrap/bin/git @@ -0,0 +1,19 @@ +#!/usr/bin/env node +// Test-only executable boundary: real Git and the real scheduler still run. +// Only a repository explicitly armed by the test can inject a timeout. +import fs from "node:fs"; +import path from "node:path"; +import { spawn } from "node:child_process"; +const args = process.argv.slice(2); +const cwd = args[0] === "-C" ? args[1] : process.cwd(); +const once = path.join(cwd, ".git", "bootstrap-fail-once"); +const always = path.join(cwd, ".git", "bootstrap-fail-always"); +if (args.includes("--ignored") && (fs.existsSync(once) || fs.existsSync(always))) { + if (fs.existsSync(once)) fs.unlinkSync(once); + setTimeout(() => process.exit(1), 60_000); +} else { + const child = spawn(process.env.BOOTSTRAP_REAL_GIT || "/usr/bin/git", args, { stdio: "inherit" }); + for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => child.kill(signal)); + child.on("exit", (code) => process.exit(code ?? 1)); + child.on("error", () => process.exit(1)); +} diff --git a/tests/e2e/workspace-bootstrap/playwright.config.ts b/tests/e2e/workspace-bootstrap/playwright.config.ts new file mode 100644 index 0000000000..18b29553f4 --- /dev/null +++ b/tests/e2e/workspace-bootstrap/playwright.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: ".", + testMatch: "*.spec.ts", + workers: 1, + // Two 30s backoffs can each wait another scheduler tick, followed by the + // three real 10s timeouts and the final no-fourth-run observation window. + timeout: 240_000, + use: { viewport: { width: 1440, height: 1080 }, screenshot: "only-on-failure", trace: "retain-on-failure" }, + outputDir: "../../../test-results/workspace-bootstrap", + reporter: [["list"]], +}); diff --git a/tests/e2e/workspace-bootstrap/recovery.spec.ts b/tests/e2e/workspace-bootstrap/recovery.spec.ts new file mode 100644 index 0000000000..90e78e4d01 --- /dev/null +++ b/tests/e2e/workspace-bootstrap/recovery.spec.ts @@ -0,0 +1,79 @@ +import { test, expect } from "@playwright/test"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { seedWorkspaceBootstrap } from "./seed.mjs"; + +// Opt-in: requires the isolated test-drive launched with this suite's Git shim. +// Never attach these fault-injection tests to a regular development instance. +const base = process.env.WORKSPACE_BOOTSTRAP_TEST_URL; +test.skip(!base, "Launch the disposable workspace-bootstrap test-drive; see README.md"); + +for (const persistent of [false, true]) { + test(persistent ? "exhausts the shared retry budget with an actionable stop" : "recovers and completes through the task UI without manual Retry", async ({ page }, info) => { + const fixture = await seedWorkspaceBootstrap(base!, persistent); + const api = async (route: string) => { + const response = await page.request.get(`${base}/api${route}`); + expect(response.ok(), await response.text()).toBeTruthy(); + return response.json(); + }; + const title = `${persistent ? "Bound persistent failure" : "Recover and preserve work"} ${Date.now()}`; + await page.goto(`${base}/${fixture.prefix}/dashboard`); + const announcement = page.getByRole("button", { name: "Dismiss announcement" }); + if (await announcement.isVisible()) await announcement.click(); + await page.getByRole("link", { name: "Tasks", exact: true }).click(); + await page.getByRole("button", { name: "New Task", exact: true }).last().click(); + await page.getByRole("textbox", { name: "Task title", exact: true }).fill(title); + await page.getByRole("button", { name: "Assignee", exact: true }).click(); + await page.getByRole("button", { name: fixture.agentName, exact: true }).click(); + // Let the closing popover unmount before clicking another popover trigger. + await expect(page.getByRole("textbox", { name: "Search assignees...", includeHidden: true })).toHaveCount(0); + // The preceding popover's focus restoration can consume the first click. + await expect(async () => { + if (!await page.getByRole("textbox", { name: "Search projects..." }).isVisible()) { + await page.getByRole("button", { name: "Project", exact: true }).click(); + } + await expect(page.getByRole("textbox", { name: "Search projects..." })).toBeVisible({ timeout: 1_000 }); + }).toPass({ timeout: 10_000, intervals: [1_000] }); + await page.getByRole("button", { name: fixture.projectName, exact: true }).click(); + await expect(page.getByRole("textbox", { name: "Search projects...", includeHidden: true })).toHaveCount(0); + await page.getByRole("button", { name: "Create Task", exact: true }).click(); + const taskLink = page.getByRole("complementary").getByRole("link", { name: title, exact: true }); + await expect(taskLink).toBeVisible(); + // Follow the UI's actual persisted link, including across task-tab state. + const href = await taskLink.getAttribute("href"); + expect(href).toBeTruthy(); + await page.goto(new URL(href!, base).href); + const tasks = await api(`/companies/${fixture.companyId}/issues`); + const task = tasks.find((row: { title: string }) => row.title === title); + const runs = async () => (await api(`/companies/${fixture.companyId}/heartbeat-runs`)).filter((row: { agentId: string }) => row.agentId === fixture.agentId); + await expect.poll(async () => (await runs()).some((row: { errorCode: string }) => row.errorCode === "workspace_git_scan_timeout"), { timeout: 30_000 }).toBe(true); + await expect(page.getByText(/Agent resumes in/)).toBeVisible(); + await page.screenshot({ path: info.outputPath("scheduled-retry.png"), fullPage: true }); + await expect.poll(async () => (await api(`/issues/${task.id}`)).status, { timeout: 150_000 }).toBe(persistent ? "blocked" : "done"); + // The worker updates the task before its process exit is persisted. + await expect.poll(async () => (await runs()).filter((row: { status: string }) => ["running", "queued", "scheduled_retry"].includes(row.status)).length).toBe(0); + const history = await runs(); + expect(history).toHaveLength(persistent ? 3 : 2); + expect(history.filter((row: { status: string }) => row.status === "succeeded")).toHaveLength(persistent ? 0 : 1); + if (persistent) { + await expect(page.getByText("Workspace scan timed out", { exact: true })).toBeVisible(); + await expect(page.getByText("No live execution path", { exact: true })).toHaveCount(0); + await page.waitForTimeout(35_000); + expect(await runs()).toHaveLength(3); + } else { + await expect(page.getByText(/Workspace recovered automatically\./)).toBeVisible(); + const project = await api(`/projects/${fixture.projectId}`); + const sourceCopy = project.workspaces.find((row: { name: string }) => row.name === "Source copy"); + const completed = history.find((row: { status: string }) => row.status === "succeeded"); + // The worker already verified the managed copy with its run-scoped API. + // Independently confirm the configured source was never overwritten. + expect(sourceCopy).toBeTruthy(); + expect(completed.scheduledRetryAttempt).toBe(1); + expect(await readFile(path.join(fixture.source, "README.md"), "utf8")).toBe("Existing uncommitted work\n"); + } + await page.reload(); + await expect(page.getByRole("button", { name: new RegExp(`^Change status \\(current: ${persistent ? "Blocked" : "Done"}`) }).first()).toBeVisible(); + await expect(page.getByText(/Agent resumes in/)).toHaveCount(0); + await page.screenshot({ path: info.outputPath("settled-task.png"), fullPage: true }); + }); +} diff --git a/tests/e2e/workspace-bootstrap/seed.mjs b/tests/e2e/workspace-bootstrap/seed.mjs new file mode 100644 index 0000000000..95831fca11 --- /dev/null +++ b/tests/e2e/workspace-bootstrap/seed.mjs @@ -0,0 +1,43 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import assert from "node:assert/strict"; + +export async function seedWorkspaceBootstrap(base, persistent = false) { + const url = new URL(base); + assert.equal(url.hostname, "127.0.0.1", "Fixture must target a disposable loopback test-drive"); + const api = async (route, method = "GET", body) => { + const response = await fetch(`${base}/api${route}`, { method, headers: { "Content-Type": "application/json" }, body: body ? JSON.stringify(body) : undefined }); + assert(response.ok, `${response.status}: ${await response.clone().text()}`); + return response.json(); + }; + const health = await api("/health"); + assert.equal(health.deploymentMode, "local_trusted"); + const [company] = await api("/companies"); + assert.equal(company.name, "Workspace Recovery QA"); + const source = await mkdtemp(path.join(os.tmpdir(), "workspace-bootstrap-source-")); + const git = (...args) => execFileSync(process.env.BOOTSTRAP_REAL_GIT || "/usr/bin/git", args, { cwd: source, stdio: "ignore" }); + git("init", "-b", "main"); + await writeFile(path.join(source, "README.md"), "Committed work\n"); + await writeFile(path.join(source, ".gitignore"), "private.secret\n"); + git("add", "."); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "seed"); + await writeFile(path.join(source, "README.md"), "Existing uncommitted work\n"); + await writeFile(path.join(source, "private.secret"), "test-only secret\n"); + await writeFile(path.join(source, ".git", persistent ? "bootstrap-fail-always" : "bootstrap-fail-once"), "armed\n"); + const suffix = path.basename(source).slice(-6); + const project = await api(`/companies/${company.id}/projects`, "POST", { name: `${persistent ? "Persistent scan failure" : "Recoverable repository"} ${suffix}`, status: "in_progress", workspace: { name: "Anchor", cwd: source, isPrimary: true } }); + await api(`/projects/${project.id}/workspaces`, "POST", { name: "Source copy", cwd: source, repoUrl: pathToFileURL(source).href, isPrimary: false }); + const agent = await api(`/companies/${company.id}/agents`, "POST", { + name: `${persistent ? "Persistent failure worker" : "Recovery worker"} ${suffix}`, role: "engineer", adapterType: "process", + adapterConfig: { command: process.execPath, args: [path.resolve(import.meta.dirname, "worker.mjs")], cwd: source }, + runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, + }); + return { base, companyId: company.id, prefix: company.issuePrefix, projectId: project.id, agentId: agent.id, agentName: agent.name, projectName: project.name, source }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + console.log(JSON.stringify(await seedWorkspaceBootstrap(process.argv[2], process.argv.includes("--persistent")))); +} diff --git a/tests/e2e/workspace-bootstrap/worker.mjs b/tests/e2e/workspace-bootstrap/worker.mjs new file mode 100644 index 0000000000..e9af0d7e41 --- /dev/null +++ b/tests/e2e/workspace-bootstrap/worker.mjs @@ -0,0 +1,22 @@ +// Deterministic process adapter, not a mocked task outcome. The worker must +// actually read the prepared repository and use its run-scoped task API. +import { readFile, readdir, writeFile, access } from "node:fs/promises"; +import path from "node:path"; +import assert from "node:assert/strict"; + +const env = process.env; +const headers = { Authorization: `Bearer ${env.PAPERCLIP_API_KEY}`, "X-Paperclip-Run-Id": env.PAPERCLIP_RUN_ID, "Content-Type": "application/json" }; +const api = async (route, method = "GET", body) => { + const response = await fetch(`${env.PAPERCLIP_API_URL}/api${route}`, { method, headers, body: body ? JSON.stringify(body) : undefined }); + assert(response.ok, `${response.status}: ${await response.clone().text()}`); + return response.json(); +}; +const run = await api(`/heartbeat-runs/${env.PAPERCLIP_RUN_ID}`); +const issueId = run.contextSnapshot.issueId; +const repositories = await readdir(path.join(process.cwd(), ".paperclip-repositories")); +const repo = path.join(process.cwd(), ".paperclip-repositories", repositories.find((name) => !name.includes(".clone-"))); +assert.equal(await readFile(path.join(repo, "README.md"), "utf8"), "Existing uncommitted work\n"); +await assert.rejects(access(path.join(repo, "private.secret"))); +await writeFile(path.join(repo, "recovery-proof.txt"), "Workspace recovered; existing work preserved; private files excluded.\n", { flag: "wx" }); +await api(`/issues/${issueId}`, "PATCH", { status: "done", comment: "Workspace recovered automatically. I read the preserved uncommitted work, verified private files were excluded, and wrote recovery-proof.txt exactly once." }); +console.log("Workspace recovery verification complete."); diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index 206129b198..eff64477a9 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -3390,6 +3390,18 @@ describe("TaskChatThread live transcript", () => { expect(tail2!.textContent).toContain("Waiting for transcript..."); }); + it("does not send a workspace bootstrap failure to connection settings", () => { + const run = { id: "workspace-prep", status: "running" as const, invocationSource: "issue", triggerDetail: null, + startedAt: "2026-09-15T10:00:00Z", finishedAt: null, createdAt: "2026-09-15T10:00:00Z", + agentId: "agent-1", agentName: "Worker", adapterType: "process" }; + render( {}} issueStatus="in_progress" activeRun={run} />); + render( {}} issueStatus="in_progress" linkedRuns={[ + { ...run, runId: run.id, status: "failed", errorCode: "workspace_git_scan_timeout", finishedAt: "2026-09-15T10:00:10Z" }, + ]} />); + expect(container.textContent).toContain("Workspace setup failed before the agent started."); + expect(container.textContent).not.toContain("Review the task’s connection"); + }); + it("renders in-flight output through TaskChatLiveTail, dropping the debug plumbing (PAP-463 C1)", () => { // Interleave the exact noise the old RunTranscriptView tail surfaced (init // row, stdout/stderr/system dumps) with real content. Only the streamed diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index 28488cfea8..e02023d782 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -1688,6 +1688,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { : "Execution was stopped before returning an answer." : code === "provider_frame_too_large" ? `Provider output exceeded the safe limit. ${retryDetail}` + : code.startsWith("workspace_git_scan_") + ? `Workspace setup failed before the agent started. ${retryDetail}` : `The runner stopped before returning an answer (${code}). ${retryDetail}`; const id = `${source.id}:failure`; entriesWithFailures.push({ @@ -2890,7 +2892,9 @@ export function TaskChatThread(props: TaskChatThreadProps) { ? liveRun.currentStatusMessage : null) || (tailStatus === "failed" - ? "This run stopped before a response was available. Review the task’s connection or recovery action below." + ? linkedRunMetaById.get(tailRunId ?? "")?.errorCode?.startsWith("workspace_git_scan_") + ? "Workspace setup failed before the agent started." + : "This run stopped before a response was available. Review the task’s connection or recovery action below." : "Waiting for transcript...") } />