diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index fb58a85b0b1..2d08420052c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -218,6 +218,7 @@ export type IssueSurfaceVisibility = (typeof ISSUE_SURFACE_VISIBILITIES)[number] export const ISSUE_RECOVERY_ACTION_KINDS = [ "missing_disposition", "stranded_assigned_issue", + "workspace_validation", "active_run_watchdog", "issue_graph_liveness", ] as const; diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 58a4e092b6a..17f01486589 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -32,6 +32,8 @@ import { issueTreeHolds, issueWorkProducts, issues, + projects, + projectWorkspaces, workspaceOperations, } from "@paperclipai/db"; import { @@ -383,6 +385,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { await db.delete(companySkills); await db.delete(workspaceOperations); await db.delete(executionWorkspaces); + await db.delete(projectWorkspaces); + await db.delete(projects); await db.delete(issuePlanDecompositions); await db.delete(issueThreadInteractions); await db.delete(documentAnnotationComments); @@ -1308,6 +1312,96 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments).toHaveLength(0); }); + it("blocks a git-sensitive local adapter before launch when a project-workspace-linked issue is missing its project id", async () => { + const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Paperclip App", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary workspace", + sourceType: "local_path", + cwd: `/tmp/paperclip-missing-workspace-${randomUUID()}`, + isPrimary: true, + }); + await db + .update(issues) + .set({ + title: "Launch from linked workspace without project id", + identifier: `${issuePrefix}-1`, + projectId: null, + projectWorkspaceId, + }) + .where(eq(issues.id, issueId)); + + const heartbeat = heartbeatService(db); + + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + const failedRun = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + expect(failedRun).toMatchObject({ + status: "failed", + errorCode: "workspace_validation_failed", + }); + expect(failedRun?.error).toContain("linked to a project workspace but has no project id"); + expect(failedRun?.resultJson).toMatchObject({ + workspaceValidation: { + reason: "missing_project_id", + adapterType: "codex_local", + issueId, + issueProjectId: null, + issueProjectWorkspaceId: projectWorkspaceId, + }, + }); + + const issue = await waitForValue(async () => + db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => { + const row = rows[0] ?? null; + return row?.status === "blocked" ? row : null; + }), + ); + expect(issue?.executionRunId).toBeNull(); + + const recoveryAction = await db + .select() + .from(issueRecoveryActions) + .where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId))) + .then((rows) => rows[0] ?? null); + expect(recoveryAction).toMatchObject({ + kind: "workspace_validation", + cause: "workspace_validation_failed", + status: "active", + ownerAgentId: agentId, + recoveryIssueId: null, + }); + expect(recoveryAction?.evidence).toMatchObject({ + sourceIssueId: issueId, + latestRunId: runId, + latestRunErrorCode: "workspace_validation_failed", + recoveryCause: "workspace_validation_failed", + }); + expect(recoveryAction?.nextAction).toContain("Repair the source issue workspace link"); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + expect(comments.some((comment) => comment.body.includes("workspace failed validation"))).toBe(true); + }); + it("queues one finish-handoff wake when a successful run leaves in-progress work without a next action", async () => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); mockAdapterExecute.mockImplementationOnce(async (ctx: { runId: string }) => { diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index d6be8c26ccd..404b777dc04 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -4,6 +4,7 @@ import { sessionCodec as codexSessionCodec } from "@paperclipai/adapter-codex-lo import { resolveDefaultAgentWorkspaceDir } from "../home-paths.js"; import { applyPersistedExecutionWorkspaceConfig, + assertGitSensitiveAdapterWorkspaceValid, buildRealizedExecutionWorkspaceFromPersisted, buildExplicitResumeSessionOverride, deriveTaskKeyWithHeartbeatFallback, @@ -38,6 +39,86 @@ function buildResolvedWorkspace(overrides: Partial = {} }; } +type WorkspaceValidationInput = Parameters[0]; + +function buildWorkspaceValidationInput( + overrides: Partial = {}, +): WorkspaceValidationInput { + return { + adapterType: "codex_local", + agentId: "agent-1", + issue: { + id: "issue-1", + identifier: "PAP-1", + projectId: "project-1", + projectWorkspaceId: "workspace-1", + }, + resolvedWorkspace: buildResolvedWorkspace(), + executionWorkspace: { + baseCwd: "/tmp/project", + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: null, + strategy: "project_primary", + cwd: "/tmp/project", + branchName: null, + worktreePath: null, + warnings: [], + created: false, + baseRefSha: null, + }, + persistedExecutionWorkspace: { + id: "execution-workspace-1", + companyId: "company-1", + projectId: "project-1", + projectWorkspaceId: "workspace-1", + sourceIssueId: "issue-1", + mode: "project_workspace", + strategyType: "project_primary", + name: "Primary workspace", + status: "active", + cwd: "/tmp/project", + repoUrl: null, + baseRef: null, + branchName: null, + providerType: "local_path", + providerRef: null, + derivedFromExecutionWorkspaceId: null, + lastUsedAt: new Date("2026-06-06T00:00:00.000Z"), + openedAt: new Date("2026-06-06T00:00:00.000Z"), + closedAt: null, + cleanupEligibleAt: null, + cleanupReason: null, + config: null, + metadata: null, + createdAt: new Date("2026-06-06T00:00:00.000Z"), + updatedAt: new Date("2026-06-06T00:00:00.000Z"), + }, + executionTarget: { kind: "local" }, + ...overrides, + }; +} + +async function expectWorkspaceValidationFailure( + input: WorkspaceValidationInput, + reason: string, + message: string, +) { + await expect(assertGitSensitiveAdapterWorkspaceValid(input)).rejects.toMatchObject({ + code: "workspace_validation_failed", + message: expect.stringContaining(message), + resultJson: { + workspaceValidation: expect.objectContaining({ + reason, + adapterType: input.adapterType, + issueId: input.issue?.id, + }), + }, + }); +} + function buildAgent(adapterType: string, runtimeConfig: Record = {}) { return { id: "agent-1", @@ -125,6 +206,173 @@ function buildIssueAncestryDb(rows: Array<{ id: string; companyId: string; paren }; } +describe("assertGitSensitiveAdapterWorkspaceValid", () => { + it("rejects a project-workspace-linked issue that is missing its project id before adapter launch", async () => { + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + issue: { + id: "issue-1", + identifier: "PAP-1", + projectId: null, + projectWorkspaceId: "workspace-1", + }, + }), + "missing_project_id", + "linked to a project workspace but has no project id", + ); + }); + + it("rejects a git-sensitive local adapter when effective cwd differs from the persisted workspace cwd", async () => { + const input = buildWorkspaceValidationInput(); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + executionWorkspace: { + ...input.executionWorkspace, + cwd: "/tmp/agent-fallback", + }, + }), + "persisted_cwd_mismatch", + 'resolved adapter cwd "/tmp/agent-fallback"', + ); + }); + + it("rejects a workspace-linked issue when no execution workspace was persisted", async () => { + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + persistedExecutionWorkspace: null, + }), + "missing_persisted_execution_workspace", + "requires a project execution workspace", + ); + }); + + it("rejects a workspace-linked issue when no effective adapter cwd was resolved", async () => { + const input = buildWorkspaceValidationInput(); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + executionWorkspace: { + ...input.executionWorkspace, + cwd: null, + }, + }), + "missing_effective_cwd", + "no adapter cwd was resolved", + ); + }); + + it("rejects a persisted execution workspace linked to a different project workspace", async () => { + const input = buildWorkspaceValidationInput(); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + projectWorkspaceId: "workspace-other", + }, + }), + "project_workspace_mismatch", + 'expected project workspace "workspace-1"', + ); + }); + + it("rejects a persisted execution workspace missing its project workspace id", async () => { + const input = buildWorkspaceValidationInput(); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + projectWorkspaceId: null, + }, + }), + "persisted_workspace_missing_project_workspace_id", + "has no project workspace id", + ); + }); + + it("rejects a workspace-linked issue that would launch from the agent fallback cwd", async () => { + const input = buildWorkspaceValidationInput(); + const fallbackCwd = resolveDefaultAgentWorkspaceDir("agent-1"); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + executionWorkspace: { + ...input.executionWorkspace, + cwd: fallbackCwd, + }, + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + cwd: fallbackCwd, + }, + }), + "fallback_agent_home_cwd", + "would launch from agent fallback cwd", + ); + }); + + it("rejects a git worktree persisted workspace when cwd differs from providerRef", async () => { + const input = buildWorkspaceValidationInput(); + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + executionWorkspace: { + ...input.executionWorkspace, + strategy: "git_worktree", + cwd: "/tmp/worktree-current", + }, + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + strategyType: "git_worktree", + cwd: "/tmp/worktree-current", + providerRef: "/tmp/worktree-expected", + }, + }), + "git_worktree_provider_ref_mismatch", + 'expected git worktree "/tmp/worktree-expected"', + ); + }); + + it("rejects a workspace-linked issue when adapter cwd has no git metadata", async () => { + const input = buildWorkspaceValidationInput(); + const cwd = "/tmp/paperclip-workspace-without-git-metadata"; + + await expectWorkspaceValidationFailure( + buildWorkspaceValidationInput({ + resolvedWorkspace: buildResolvedWorkspace({ cwd }), + executionWorkspace: { + ...input.executionWorkspace, + baseCwd: cwd, + cwd, + }, + persistedExecutionWorkspace: { + ...input.persistedExecutionWorkspace!, + cwd, + }, + }), + "missing_git_metadata", + "has no .git metadata", + ); + }); + + it("does not apply the git-sensitive workspace guard to non-local execution targets", async () => { + const input = buildWorkspaceValidationInput(); + + await expect( + assertGitSensitiveAdapterWorkspaceValid( + buildWorkspaceValidationInput({ + executionTarget: { kind: "cloud" }, + executionWorkspace: { + ...input.executionWorkspace, + cwd: "/tmp/agent-fallback", + }, + }), + ), + ).resolves.toBeUndefined(); + }); +}); + describe("stripHostWorkspaceProvisionForLowTrustSandbox", () => { it("removes only the host-side provision command for sandbox-backed low-trust runs", () => { const config = { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 9e72e6fbcc2..076f53c5eb7 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -240,6 +240,20 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; +const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed"; +const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; +// Keep this in sync with local adapters that require a git workspace before launch. +const GIT_SENSITIVE_LOCAL_ADAPTER_TYPES = new Set([ + "acpx_local", + "claude_local", + "codex_local", + "cursor", + "gemini_local", + "grok_local", + "hermes_local", + "opencode_local", + "pi_local", +]); export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation"; export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry"; const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; @@ -259,6 +273,17 @@ interface MaxTurnContinuationPolicy { delayMs: number; } +export class WorkspaceValidationFailure extends Error { + code = WORKSPACE_VALIDATION_FAILURE_CODE; + resultJson: Record; + + constructor(message: string, resultJson: Record) { + super(message); + this.name = "WorkspaceValidationFailure"; + this.resultJson = resultJson; + } +} + function resolveCodexTransientFallbackMode(attempt: number): CodexTransientFallbackMode { if (attempt <= 1) return "same_session"; if (attempt === 2) return "safer_invocation"; @@ -891,6 +916,184 @@ async function ensureManagedProjectWorkspace(input: { } } +function isWorkspaceValidationFailure(error: unknown): error is WorkspaceValidationFailure { + return error instanceof WorkspaceValidationFailure; +} + +function isWorkspaceValidationFailedRun( + run: Pick | null | undefined, +) { + return run?.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE; +} + +async function hasGitMetadata(cwd: string | null | undefined) { + const normalized = readNonEmptyString(cwd); + if (!normalized) return false; + return fs + .lstat(path.resolve(normalized, ".git")) + .then((entry) => entry.isDirectory() || entry.isFile()) + .catch(() => false); +} + +function sameResolvedPath(left: string | null | undefined, right: string | null | undefined) { + const leftPath = readNonEmptyString(left); + const rightPath = readNonEmptyString(right); + if (!leftPath || !rightPath) return false; + return path.resolve(leftPath) === path.resolve(rightPath); +} + +export async function assertGitSensitiveAdapterWorkspaceValid(input: { + adapterType: string; + agentId: string; + issue: { + id: string; + identifier: string | null; + projectId: string | null; + projectWorkspaceId: string | null; + } | null; + resolvedWorkspace: ResolvedWorkspaceForRun; + executionWorkspace: RealizedExecutionWorkspace; + persistedExecutionWorkspace: ExecutionWorkspace | null; + executionTarget: unknown; + environmentDriver?: string | null; + leaseMetadata?: unknown; +}) { + if (!GIT_SENSITIVE_LOCAL_ADAPTER_TYPES.has(input.adapterType)) return; + + const executionTargetKind = readNonEmptyString((input.executionTarget as { kind?: unknown } | null)?.kind) ?? "local"; + if (executionTargetKind !== "local") return; + + const issue = input.issue; + if (!issue) return; + + const environmentDriver = readNonEmptyString(input.environmentDriver) ?? "local"; + const leaseMetadata = parseObject(input.leaseMetadata); + const leaseProviderMetadata = parseObject(leaseMetadata.providerMetadata); + const leaseRemoteCwd = + readNonEmptyString(leaseMetadata.remoteCwd) ?? + readNonEmptyString(leaseProviderMetadata.remoteCwd); + + const effectiveCwd = readNonEmptyString(input.executionWorkspace.cwd); + const persistedCwd = readNonEmptyString(input.persistedExecutionWorkspace?.cwd); + const agentFallbackCwd = resolveDefaultAgentWorkspaceDir(input.agentId); + const workspaceExpectation = + Boolean(issue.projectWorkspaceId) || + Boolean(input.resolvedWorkspace.workspaceId) || + input.executionWorkspace.strategy === "git_worktree"; + + const fail = (reason: string, message: string, extra: Record = {}) => { + throw new WorkspaceValidationFailure(message, { + workspaceValidation: { + reason, + adapterType: input.adapterType, + issueId: issue.id, + issueIdentifier: issue.identifier, + issueProjectId: issue.projectId, + issueProjectWorkspaceId: issue.projectWorkspaceId, + resolvedWorkspaceSource: input.resolvedWorkspace.source, + resolvedProjectId: input.resolvedWorkspace.projectId, + resolvedProjectWorkspaceId: input.resolvedWorkspace.workspaceId, + resolvedWorkspaceCwd: input.resolvedWorkspace.cwd, + executionWorkspaceCwd: effectiveCwd, + executionWorkspaceStrategy: input.executionWorkspace.strategy, + executionWorkspaceProjectId: input.executionWorkspace.projectId, + executionWorkspaceProjectWorkspaceId: input.executionWorkspace.workspaceId, + persistedExecutionWorkspaceId: input.persistedExecutionWorkspace?.id ?? null, + persistedWorkspaceCwd: persistedCwd, + persistedWorkspaceStrategy: input.persistedExecutionWorkspace?.strategyType ?? null, + persistedProjectId: input.persistedExecutionWorkspace?.projectId ?? null, + persistedProjectWorkspaceId: input.persistedExecutionWorkspace?.projectWorkspaceId ?? null, + persistedProviderRef: input.persistedExecutionWorkspace?.providerRef ?? null, + ...extra, + }, + }); + }; + + if (issue.projectWorkspaceId && !issue.projectId) { + fail( + "missing_project_id", + `Issue ${issue.identifier ?? issue.id} is linked to a project workspace but has no project id; refusing to launch ${input.adapterType} from fallback cwd.`, + ); + } + + if (!input.executionTarget && environmentDriver !== "local" && leaseRemoteCwd) return; + + if (workspaceExpectation && !input.persistedExecutionWorkspace) { + fail( + "missing_persisted_execution_workspace", + `Issue ${issue.identifier ?? issue.id} requires a project execution workspace, but none was persisted before adapter launch.`, + ); + } + + if (workspaceExpectation && !effectiveCwd) { + fail( + "missing_effective_cwd", + `Issue ${issue.identifier ?? issue.id} expected a project workspace, but no adapter cwd was resolved before launch.`, + ); + } + + if ( + input.persistedExecutionWorkspace && + effectiveCwd && + persistedCwd && + !sameResolvedPath(effectiveCwd, persistedCwd) + ) { + fail( + "persisted_cwd_mismatch", + `Issue ${issue.identifier ?? issue.id} resolved adapter cwd "${effectiveCwd}" but persisted execution workspace cwd is "${persistedCwd}".`, + ); + } + + const expectedProjectWorkspaceId = issue.projectWorkspaceId ?? input.resolvedWorkspace.workspaceId ?? null; + if ( + expectedProjectWorkspaceId && + input.persistedExecutionWorkspace && + !input.persistedExecutionWorkspace.projectWorkspaceId + ) { + fail( + "persisted_workspace_missing_project_workspace_id", + `Issue ${issue.identifier ?? issue.id} expected project workspace "${expectedProjectWorkspaceId}" but persisted execution workspace has no project workspace id.`, + ); + } + + if ( + expectedProjectWorkspaceId && + input.persistedExecutionWorkspace?.projectWorkspaceId && + input.persistedExecutionWorkspace.projectWorkspaceId !== expectedProjectWorkspaceId + ) { + fail( + "project_workspace_mismatch", + `Issue ${issue.identifier ?? issue.id} expected project workspace "${expectedProjectWorkspaceId}" but persisted execution workspace points at "${input.persistedExecutionWorkspace.projectWorkspaceId}".`, + ); + } + + if (workspaceExpectation && effectiveCwd && sameResolvedPath(effectiveCwd, agentFallbackCwd)) { + fail( + "fallback_agent_home_cwd", + `Issue ${issue.identifier ?? issue.id} expected a project workspace, but ${input.adapterType} would launch from agent fallback cwd "${effectiveCwd}".`, + ); + } + + if ( + input.persistedExecutionWorkspace?.strategyType === "git_worktree" && + input.persistedExecutionWorkspace.providerRef && + effectiveCwd && + !sameResolvedPath(effectiveCwd, input.persistedExecutionWorkspace.providerRef) + ) { + fail( + "git_worktree_provider_ref_mismatch", + `Issue ${issue.identifier ?? issue.id} expected git worktree "${input.persistedExecutionWorkspace.providerRef}" but adapter cwd resolved to "${effectiveCwd}".`, + ); + } + + if (workspaceExpectation && effectiveCwd && !await hasGitMetadata(effectiveCwd)) { + fail( + "missing_git_metadata", + `Issue ${issue.identifier ?? issue.id} expected a git workspace for ${input.adapterType}, but "${effectiveCwd}" has no .git metadata.`, + ); + } +} + const heartbeatRunProcessGroupIdColumn = heartbeatRuns.processGroupId ?? sql`NULL`.as("processGroupId"); @@ -8314,6 +8517,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const logEntry = formatRuntimeWorkspaceWarningLog(warning); await onLog(logEntry.stream, logEntry.chunk); } + await assertGitSensitiveAdapterWorkspaceValid({ + adapterType: agent.adapterType, + agentId: agent.id, + issue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + projectId: issueRef.projectId, + projectWorkspaceId: issueRef.projectWorkspaceId, + } + : null, + resolvedWorkspace, + executionWorkspace, + persistedExecutionWorkspace, + executionTarget, + environmentDriver: selectedEnvironment.driver, + leaseMetadata: activeEnvironmentLease.lease.metadata, + }); const adapterEnv = Object.fromEntries( Object.entries(parseObject(resolvedConfig.env)).filter( (entry): entry is [string, string] => typeof entry[0] === "string" && typeof entry[1] === "string", @@ -8798,6 +9019,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions(), ); + const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err : null; + const failureErrorCode = workspaceValidationFailure?.code ?? "adapter_failed"; logger.error({ err, runId }, "heartbeat execution failed"); let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null; @@ -8818,11 +9041,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const failedRun = await setRunStatus(run.id, "failed", { error: message, - errorCode: "adapter_failed", + errorCode: failureErrorCode, finishedAt: new Date(), resultJson: mergeRunStopMetadataForAgent(agent, "failed", { - errorCode: "adapter_failed", + errorCode: failureErrorCode, errorMessage: message, + resultJson: workspaceValidationFailure?.resultJson ?? null, }), stdoutExcerpt, stderrExcerpt, @@ -8844,7 +9068,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun; await refreshContinuationSummaryForRun(livenessRun, agent); - await finalizeIssueCommentPolicy(livenessRun, agent); + if (!isWorkspaceValidationFailedRun(livenessRun)) { + await finalizeIssueCommentPolicy(livenessRun, agent); + } await releaseIssueExecutionAndPromote(livenessRun); await updateRuntimeState(agent, livenessRun, { @@ -8907,7 +9133,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null); if (failedAgent) { await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined); - await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); + if (!isWorkspaceValidationFailedRun(livenessRun)) { + await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined); + } } await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined); } @@ -8949,6 +9177,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); } + function buildWorkspaceValidationRecoveryComment(input: { + latestRun: Pick | null | undefined; + }) { + const failureSummary = summarizeRunFailureForIssueComment(input.latestRun); + return ( + "Paperclip stopped before launching the local adapter because the issue workspace failed validation. " + + `This prevents git-sensitive adapters from running in an unrelated fallback cwd.${failureSummary ?? ""} ` + + "Moving it to `blocked` with a source-scoped recovery action so the workspace link, cwd, or git checkout can be repaired before resuming." + ); + } + async function releaseIssueExecutionAndPromote(run: typeof heartbeatRuns.$inferSelect) { const runContext = parseObject(run.contextSnapshot); const contextIssueId = readNonEmptyString(runContext.issueId); @@ -9001,6 +9240,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where(eq(issues.id, issue.id)); } + if ( + isWorkspaceValidationFailedRun(run) && + (issue.status === "todo" || issue.status === "in_progress") && + !issue.assigneeUserId && + issue.assigneeAgentId === run.agentId + ) { + return { + kind: "blocked" as const, + issue, + previousStatus: issue.status, + comment: buildWorkspaceValidationRecoveryComment({ latestRun: run }), + recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE, + }; + } + while (true) { const deferred = await tx .select() @@ -9247,17 +9501,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const shouldBlockImmediately = !recoveryAgentInvokable || !recoveryAgent || + isWorkspaceValidationFailedRun(run) || didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed"); if (shouldBlockImmediately) { - const comment = buildImmediateExecutionPathRecoveryComment({ - status: issue.status as "todo" | "in_progress", - latestRun: run, - }); + const workspaceValidationFailure = isWorkspaceValidationFailedRun(run); + const comment = workspaceValidationFailure + ? buildWorkspaceValidationRecoveryComment({ latestRun: run }) + : buildImmediateExecutionPathRecoveryComment({ + status: issue.status as "todo" | "in_progress", + latestRun: run, + }); return { kind: "blocked" as const, issue, previousStatus: issue.status, comment, + recoveryCause: workspaceValidationFailure ? WORKSPACE_VALIDATION_RECOVERY_CAUSE : undefined, }; } @@ -9340,6 +9599,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) previousStatus: promotionResult.previousStatus as "todo" | "in_progress", latestRun: run, comment: promotionResult.comment, + recoveryCause: + promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE + ? WORKSPACE_VALIDATION_RECOVERY_CAUSE + : undefined, }); return; } diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 03511e68f4f..b7f91508a61 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -5040,21 +5040,29 @@ export function issueService(db: Db) { issueData.executionWorkspaceSettings !== undefined ? parseIssueExecutionWorkspaceSettings(issueData.executionWorkspaceSettings) : parseIssueExecutionWorkspaceSettings(existing.executionWorkspaceSettings); + let validatedProjectWorkspace: { projectId: string } | null = null; + let validatedExecutionWorkspace: { projectId: string } | null = null; if (!nextProjectId && nextProjectWorkspaceId) { const workspace = await assertValidProjectWorkspace(existing.companyId, null, nextProjectWorkspaceId); + validatedProjectWorkspace = workspace; nextProjectId = workspace.projectId; patch.projectId = workspace.projectId; } if (!nextProjectId && nextExecutionWorkspaceId) { const workspace = await assertValidExecutionWorkspace(existing.companyId, null, nextExecutionWorkspaceId); + validatedExecutionWorkspace = workspace; nextProjectId = workspace.projectId; patch.projectId = workspace.projectId; } if (nextProjectWorkspaceId) { - await assertValidProjectWorkspace(existing.companyId, nextProjectId, nextProjectWorkspaceId); + if (!validatedProjectWorkspace) { + await assertValidProjectWorkspace(existing.companyId, nextProjectId, nextProjectWorkspaceId); + } } if (nextExecutionWorkspaceId) { - await assertValidExecutionWorkspace(existing.companyId, nextProjectId, nextExecutionWorkspaceId); + if (!validatedExecutionWorkspace) { + await assertValidExecutionWorkspace(existing.companyId, nextProjectId, nextExecutionWorkspaceId); + } } applyStatusSideEffects(issueData.status, patch); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 77e9d3fbdfd..468d76c9c8f 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -102,7 +102,10 @@ type LatestIssueRun = Pick< > | null; type SuccessfulLatestIssueRun = NonNullable & { status: "succeeded" }; -type StrandedRecoveryCause = "stranded_assigned_issue" | typeof SUCCESSFUL_RUN_MISSING_STATE_REASON; +type StrandedRecoveryCause = + | "stranded_assigned_issue" + | "workspace_validation_failed" + | typeof SUCCESSFUL_RUN_MISSING_STATE_REASON; type SuccessfulRunHandoffRecoveryEvidence = { sourceRunId: string | null; @@ -2036,6 +2039,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) function strandedRecoveryActionKind(cause: StrandedRecoveryCause) { return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON ? "missing_disposition" as const + : cause === "workspace_validation_failed" + ? "workspace_validation" as const : "stranded_assigned_issue" as const; } @@ -2109,8 +2114,16 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) }), nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON ? "Choose and record a valid issue disposition without copying transcript content." + : recoveryCause === "workspace_validation_failed" + ? "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution." : "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.", - wakePolicy: ownerAgentId + wakePolicy: recoveryCause === "workspace_validation_failed" + ? { + type: "manual_repair_required", + reason: "workspace_validation_failed", + ownerAgentId, + } + : ownerAgentId ? { type: "wake_owner", reason: "source_scoped_recovery_action", @@ -2134,6 +2147,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: LatestIssueRun; recoveryCause: StrandedRecoveryCause; }) { + if (input.recoveryCause === "workspace_validation_failed") return; if (!input.action.ownerAgentId) return; await deps.enqueueWakeup(input.action.ownerAgentId, { source: "assignment", @@ -2337,7 +2351,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) "- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.", ].join("\n"); - if (recoveryAction.attemptCount === 1) { + const shouldPostEscalationComment = + recoveryAction.attemptCount === 1 || + input.recoveryCause === "workspace_validation_failed"; + if (shouldPostEscalationComment) { const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``; const hasEscalationComment = await db @@ -2388,6 +2405,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) previousStatus: input.previousStatus, source: input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON ? "recovery.reconcile_successful_run_handoff_missing_state" + : input.recoveryCause === "workspace_validation_failed" + ? "recovery.reconcile_workspace_validation_failed" : "recovery.reconcile_stranded_assigned_issue", recoveryCause: input.recoveryCause ?? "stranded_assigned_issue", latestRunId: input.latestRun?.id ?? null, diff --git a/ui/src/components/IssueBlockedNotice.test.tsx b/ui/src/components/IssueBlockedNotice.test.tsx index 27f93a0b8ed..3eb63ae3ce1 100644 --- a/ui/src/components/IssueBlockedNotice.test.tsx +++ b/ui/src/components/IssueBlockedNotice.test.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom -import { act } from "react"; import { createRoot } from "react-dom/client"; +import { flushSync } from "react-dom"; import type { AnchorHTMLAttributes, ReactElement, ReactNode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -27,6 +27,18 @@ vi.mock("../api/issues", () => ({ // eslint-disable-next-line @typescript-eslint/no-explicit-any (globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; +function act(callback: () => T): T { + let result: T | undefined; + flushSync(() => { + result = callback(); + }); + const maybePromise = result as unknown as PromiseLike; + if (result && typeof maybePromise.then === "function") { + throw new TypeError("This test act shim only supports synchronous callbacks."); + } + return result as T; +} + let root: ReturnType | null = null; let container: HTMLDivElement | null = null; let dateNowSpy: ReturnType | null = null; @@ -158,9 +170,8 @@ describe("IssueBlockedNotice", () => { expect(button).not.toBeNull(); expect(button!.textContent ?? "").toContain("Retry now"); - await act(async () => { + act(() => { button!.click(); - await Promise.resolve(); }); await vi.waitFor(() => { @@ -263,6 +274,65 @@ describe("IssueBlockedNotice", () => { ); expect(indicator).not.toBeNull(); expect(indicator?.getAttribute("data-recovery-state")).toBe("needed"); + expect(indicator?.getAttribute("data-recovery-kind")).toBe("missing_disposition"); expect(indicator?.textContent).toContain("Recovery needed"); }); + + it("labels a workspace_validation blocker recovery distinctly", () => { + const node = render( + , + ); + + const indicator = node.querySelector( + '[data-testid="issue-blocked-notice-recovery-indicator"]', + ); + expect(indicator).not.toBeNull(); + expect(indicator?.getAttribute("data-recovery-state")).toBe("needed"); + expect(indicator?.getAttribute("data-recovery-kind")).toBe("workspace_validation"); + expect(indicator?.textContent).toContain("Workspace recovery needed"); + }); }); diff --git a/ui/src/components/IssueBlockedNotice.tsx b/ui/src/components/IssueBlockedNotice.tsx index 0ec0eb4fea1..8182f997c40 100644 --- a/ui/src/components/IssueBlockedNotice.tsx +++ b/ui/src/components/IssueBlockedNotice.tsx @@ -17,6 +17,7 @@ import { isAssignedBacklogBlocker } from "../lib/issue-blockers"; import { deriveActiveRecoveryDisplayState, RECOVERY_CHIP_DEFAULT_TONE, + recoveryChipLabel, } from "../lib/recovery-display"; function BlockerRecoveryIndicator({ action }: { action: IssueRecoveryAction }) { @@ -24,17 +25,19 @@ function BlockerRecoveryIndicator({ action }: { action: IssueRecoveryAction }) { if (!state) return null; const tone = RECOVERY_CHIP_DEFAULT_TONE[state]; const Icon = tone.icon; + const label = recoveryChipLabel(state, action.kind); return ( - {tone.label} + {label} ); } diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index 5b80ceca439..ea280927fe5 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom -import { act } from "react"; import { createRoot } from "react-dom/client"; +import { flushSync } from "react-dom"; import type { AnchorHTMLAttributes, ReactElement } from "react"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { Agent, IssueRecoveryAction } from "@paperclipai/shared"; @@ -16,6 +16,18 @@ vi.mock("@/lib/router", () => ({ // eslint-disable-next-line @typescript-eslint/no-explicit-any (globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; +function act(callback: () => T): T { + let result: T | undefined; + flushSync(() => { + result = callback(); + }); + const maybePromise = result as unknown as PromiseLike; + if (result && typeof maybePromise.then === "function") { + throw new TypeError("This test act shim only supports synchronous callbacks."); + } + return result as T; +} + let root: ReturnType | null = null; let container: HTMLDivElement | null = null; @@ -157,6 +169,33 @@ describe("IssueRecoveryActionCard", () => { expect(node.textContent).toContain("OBSERVING ACTIVE RUN"); }); + it("renders a workspace-specific label and headline for workspace_validation", () => { + const node = render( + , + ); + const section = node.querySelector("section[aria-label]"); + expect(section?.getAttribute("data-recovery-kind")).toBe("workspace_validation"); + expect(node.textContent).toContain("Workspace Validation"); + expect(node.textContent).not.toContain("workspace_validation\n"); + expect(node.textContent).toContain( + "Paperclip stopped this run because the issue's git workspace could not be validated.", + ); + expect(node.textContent).toContain("Repair the source issue workspace link"); + expect(node.textContent).toContain("Manual repair required"); + }); + it("renders the resolved label and outcome when resolved", () => { const node = render( , diff --git a/ui/src/components/IssueRecoveryActionCard.tsx b/ui/src/components/IssueRecoveryActionCard.tsx index 16ce3f5f194..8fa0663ec50 100644 --- a/ui/src/components/IssueRecoveryActionCard.tsx +++ b/ui/src/components/IssueRecoveryActionCard.tsx @@ -46,6 +46,7 @@ export interface IssueRecoveryActionCardProps { const KIND_LABEL: Record = { missing_disposition: "Missing Disposition", stranded_assigned_issue: "Stranded Issue", + workspace_validation: "Workspace Validation", active_run_watchdog: "Active Watchdog", issue_graph_liveness: "Graph Liveness", }; @@ -54,6 +55,8 @@ const KIND_HEADLINE: Record = { missing_disposition: "This issue's run finished, but no next step was chosen.", stranded_assigned_issue: "Paperclip retried this issue's last run and it still has no live execution path.", + workspace_validation: + "Paperclip stopped this run because the issue's git workspace could not be validated.", active_run_watchdog: "The active run has been silent. Recovery is observing without interrupting it.", issue_graph_liveness: @@ -169,6 +172,7 @@ function readWakePolicySummary(action: IssueRecoveryAction): string | null { if (type === "wake_owner") return "Corrective wake queued"; if (type === "board_escalation") return "Escalated to board"; if (type === "manual") return "Manual"; + if (type === "manual_repair_required") return "Manual repair required"; if (type === "monitor") { const interval = readEvidenceString(policy.intervalLabel); return interval ? `Monitor scheduled · ${interval}` : "Monitor scheduled"; diff --git a/ui/src/components/IssueRow.tsx b/ui/src/components/IssueRow.tsx index d2d505146e3..578dc9ad5ed 100644 --- a/ui/src/components/IssueRow.tsx +++ b/ui/src/components/IssueRow.tsx @@ -8,7 +8,11 @@ import { withIssueDetailHeaderSeed, } from "../lib/issueDetailBreadcrumb"; import { cn } from "../lib/utils"; -import { deriveActiveRecoveryDisplayState, RECOVERY_CHIP_DEFAULT_TONE } from "../lib/recovery-display"; +import { + deriveActiveRecoveryDisplayState, + RECOVERY_CHIP_DEFAULT_TONE, + recoveryChipLabel, +} from "../lib/recovery-display"; import { StatusIcon } from "./StatusIcon"; import { productivityReviewTriggerLabel } from "./ProductivityReviewBadge"; import { hasAssignedBacklogBlocker } from "../lib/issue-blockers"; @@ -233,21 +237,23 @@ function renderRecoveryChip(action: IssueRecoveryAction, selected: boolean): Rea if (!state) return null; const tone = RECOVERY_CHIP_DEFAULT_TONE[state]; const Icon = tone.icon; + const label = recoveryChipLabel(state, action.kind); return ( - {tone.label} + {label} ); } diff --git a/ui/src/lib/recovery-display.test.ts b/ui/src/lib/recovery-display.test.ts new file mode 100644 index 00000000000..63235714bce --- /dev/null +++ b/ui/src/lib/recovery-display.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + deriveActiveRecoveryDisplayState, + deriveRecoveryDisplayState, + recoveryChipLabel, +} from "./recovery-display"; + +describe("recoveryChipLabel", () => { + it("returns the workspace-specific label when kind is workspace_validation and state is needed", () => { + expect(recoveryChipLabel("needed", "workspace_validation")).toBe( + "Workspace recovery needed", + ); + }); + + it("falls back to the generic label for other needed kinds", () => { + expect(recoveryChipLabel("needed", "missing_disposition")).toBe("Recovery needed"); + expect(recoveryChipLabel("needed", "stranded_assigned_issue")).toBe("Recovery needed"); + expect(recoveryChipLabel("needed", "issue_graph_liveness")).toBe("Recovery needed"); + }); + + it("does not override the chip label for non-needed states", () => { + expect(recoveryChipLabel("in_progress", "workspace_validation")).toBe( + "Recovery in progress", + ); + expect(recoveryChipLabel("escalated", "workspace_validation")).toBe( + "Recovery escalated", + ); + expect(recoveryChipLabel("observe_only", "workspace_validation")).toBe( + "Observing active run", + ); + }); +}); + +describe("deriveRecoveryDisplayState", () => { + const base = { + status: "active" as const, + kind: "missing_disposition" as const, + outcome: null, + }; + + it("classifies workspace_validation active as needed", () => { + expect(deriveRecoveryDisplayState({ ...base, kind: "workspace_validation" })).toBe( + "needed", + ); + expect( + deriveActiveRecoveryDisplayState({ ...base, kind: "workspace_validation" }), + ).toBe("needed"); + }); +}); diff --git a/ui/src/lib/recovery-display.ts b/ui/src/lib/recovery-display.ts index 8a5dc346e59..b0f3a92a581 100644 --- a/ui/src/lib/recovery-display.ts +++ b/ui/src/lib/recovery-display.ts @@ -1,4 +1,4 @@ -import type { IssueRecoveryAction } from "@paperclipai/shared"; +import type { IssueRecoveryAction, IssueRecoveryActionKind } from "@paperclipai/shared"; import { Eye, OctagonAlert, RefreshCw, TriangleAlert } from "lucide-react"; export type RecoveryDisplayState = @@ -55,3 +55,13 @@ export function deriveActiveRecoveryDisplayState( const state = deriveRecoveryDisplayState(action); return state === "resolved" ? null : state; } + +export function recoveryChipLabel( + state: ActiveRecoveryDisplayState, + kind: IssueRecoveryActionKind, +): string { + if (kind === "workspace_validation" && state === "needed") { + return "Workspace recovery needed"; + } + return RECOVERY_CHIP_DEFAULT_TONE[state].label; +} diff --git a/ui/storybook/stories/source-issue-recovery.stories.tsx b/ui/storybook/stories/source-issue-recovery.stories.tsx index fd6257f5177..f252b159191 100644 --- a/ui/storybook/stories/source-issue-recovery.stories.tsx +++ b/ui/storybook/stories/source-issue-recovery.stories.tsx @@ -129,6 +129,21 @@ function AllStatesPanel() { nextAction: "Issue restored to a valid disposition.", })} /> + ); } @@ -173,6 +188,20 @@ function BlockerNoticePanel() { identifier: "PAP-9051", title: "Bare blocker without recovery state", }), + buildBlocker({ + id: "blocker-5", + identifier: "PAP-10409", + title: "Phase 6 follow-up failed: git workspace lost", + status: "blocked", + activeRecoveryAction: buildAction({ + kind: "workspace_validation", + cause: "workspace_validation_failed", + fingerprint: "fp-workspace-blocker", + evidence: { latestRunErrorCode: "workspace_validation_failed" }, + nextAction: + "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.", + }), + }), ]} /> @@ -322,6 +351,23 @@ function InboxRowPanel() { activeRecoveryAction: buildAction({ status: "escalated" }), }} /> + ); }