PAP-10419 guard git-sensitive adapter workspaces

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-06-06 13:37:08 +00:00
1 parent 81512f07c8
commit 969594e7ae
14 files changed
+885 -25

No files matched your search

+1
View File
@@ -218,6 +218,7 @@ export type IssueSurfaceVisibility = (typeof ISSUE_SURFACE_VISIBILITIES)[number]
export const ISSUE_RECOVERY_ACTION_KINDS = [
"missing_disposition",
"stranded_assigned_issue",
"workspace_validation",
"active_run_watchdog",
"issue_graph_liveness",
] as const;
@@ -32,6 +32,8 @@ import {
issueTreeHolds,
issueWorkProducts,
issues,
projects,
projectWorkspaces,
workspaceOperations,
} from "@paperclipai/db";
import {
@@ -383,6 +385,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await db.delete(companySkills);
await db.delete(workspaceOperations);
await db.delete(executionWorkspaces);
await db.delete(projectWorkspaces);
await db.delete(projects);
await db.delete(issuePlanDecompositions);
await db.delete(issueThreadInteractions);
await db.delete(documentAnnotationComments);
@@ -1308,6 +1312,96 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(comments).toHaveLength(0);
});
it("blocks a git-sensitive local adapter before launch when a project-workspace-linked issue is missing its project id", async () => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
const projectId = randomUUID();
const projectWorkspaceId = randomUUID();
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
await db.insert(projects).values({
id: projectId,
companyId,
name: "Paperclip App",
status: "in_progress",
});
await db.insert(projectWorkspaces).values({
id: projectWorkspaceId,
companyId,
projectId,
name: "Primary workspace",
sourceType: "local_path",
cwd: `/tmp/paperclip-missing-workspace-${randomUUID()}`,
isPrimary: true,
});
await db
.update(issues)
.set({
title: "Launch from linked workspace without project id",
identifier: `${issuePrefix}-1`,
projectId: null,
projectWorkspaceId,
})
.where(eq(issues.id, issueId));
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
expect(mockAdapterExecute).not.toHaveBeenCalled();
const failedRun = await db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
expect(failedRun).toMatchObject({
status: "failed",
errorCode: "workspace_validation_failed",
});
expect(failedRun?.error).toContain("linked to a project workspace but has no project id");
expect(failedRun?.resultJson).toMatchObject({
workspaceValidation: {
reason: "missing_project_id",
adapterType: "codex_local",
issueId,
issueProjectId: null,
issueProjectWorkspaceId: projectWorkspaceId,
},
});
const issue = await waitForValue(async () =>
db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => {
const row = rows[0] ?? null;
return row?.status === "blocked" ? row : null;
}),
);
expect(issue?.executionRunId).toBeNull();
const recoveryAction = await db
.select()
.from(issueRecoveryActions)
.where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId)))
.then((rows) => rows[0] ?? null);
expect(recoveryAction).toMatchObject({
kind: "workspace_validation",
cause: "workspace_validation_failed",
status: "active",
ownerAgentId: agentId,
recoveryIssueId: null,
});
expect(recoveryAction?.evidence).toMatchObject({
sourceIssueId: issueId,
latestRunId: runId,
latestRunErrorCode: "workspace_validation_failed",
recoveryCause: "workspace_validation_failed",
});
expect(recoveryAction?.nextAction).toContain("Repair the source issue workspace link");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments.some((comment) => comment.body.includes("workspace failed validation"))).toBe(true);
});
it("queues one finish-handoff wake when a successful run leaves in-progress work without a next action", async () => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
mockAdapterExecute.mockImplementationOnce(async (ctx: { runId: string }) => {
@@ -4,6 +4,7 @@ import { sessionCodec as codexSessionCodec } from "@paperclipai/adapter-codex-lo
import { resolveDefaultAgentWorkspaceDir } from "../home-paths.js";
import {
applyPersistedExecutionWorkspaceConfig,
assertGitSensitiveAdapterWorkspaceValid,
buildRealizedExecutionWorkspaceFromPersisted,
buildExplicitResumeSessionOverride,
deriveTaskKeyWithHeartbeatFallback,
@@ -38,6 +39,86 @@ function buildResolvedWorkspace(overrides: Partial<ResolvedWorkspaceForRun> = {}
};
}
type WorkspaceValidationInput = Parameters<typeof assertGitSensitiveAdapterWorkspaceValid>[0];
function buildWorkspaceValidationInput(
overrides: Partial<WorkspaceValidationInput> = {},
): WorkspaceValidationInput {
return {
adapterType: "codex_local",
agentId: "agent-1",
issue: {
id: "issue-1",
identifier: "PAP-1",
projectId: "project-1",
projectWorkspaceId: "workspace-1",
},
resolvedWorkspace: buildResolvedWorkspace(),
executionWorkspace: {
baseCwd: "/tmp/project",
source: "project_primary",
projectId: "project-1",
workspaceId: "workspace-1",
repoUrl: null,
repoRef: null,
strategy: "project_primary",
cwd: "/tmp/project",
branchName: null,
worktreePath: null,
warnings: [],
created: false,
baseRefSha: null,
},
persistedExecutionWorkspace: {
id: "execution-workspace-1",
companyId: "company-1",
projectId: "project-1",
projectWorkspaceId: "workspace-1",
sourceIssueId: "issue-1",
mode: "project_workspace",
strategyType: "project_primary",
name: "Primary workspace",
status: "active",
cwd: "/tmp/project",
repoUrl: null,
baseRef: null,
branchName: null,
providerType: "local_path",
providerRef: null,
derivedFromExecutionWorkspaceId: null,
lastUsedAt: new Date("2026-06-06T00:00:00.000Z"),
openedAt: new Date("2026-06-06T00:00:00.000Z"),
closedAt: null,
cleanupEligibleAt: null,
cleanupReason: null,
config: null,
metadata: null,
createdAt: new Date("2026-06-06T00:00:00.000Z"),
updatedAt: new Date("2026-06-06T00:00:00.000Z"),
},
executionTarget: { kind: "local" },
...overrides,
};
}
async function expectWorkspaceValidationFailure(
input: WorkspaceValidationInput,
reason: string,
message: string,
) {
await expect(assertGitSensitiveAdapterWorkspaceValid(input)).rejects.toMatchObject({
code: "workspace_validation_failed",
message: expect.stringContaining(message),
resultJson: {
workspaceValidation: expect.objectContaining({
reason,
adapterType: input.adapterType,
issueId: input.issue?.id,
}),
},
});
}
function buildAgent(adapterType: string, runtimeConfig: Record<string, unknown> = {}) {
return {
id: "agent-1",
@@ -125,6 +206,173 @@ function buildIssueAncestryDb(rows: Array<{ id: string; companyId: string; paren
};
}
describe("assertGitSensitiveAdapterWorkspaceValid", () => {
it("rejects a project-workspace-linked issue that is missing its project id before adapter launch", async () => {
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
issue: {
id: "issue-1",
identifier: "PAP-1",
projectId: null,
projectWorkspaceId: "workspace-1",
},
}),
"missing_project_id",
"linked to a project workspace but has no project id",
);
});
it("rejects a git-sensitive local adapter when effective cwd differs from the persisted workspace cwd", async () => {
const input = buildWorkspaceValidationInput();
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
executionWorkspace: {
...input.executionWorkspace,
cwd: "/tmp/agent-fallback",
},
}),
"persisted_cwd_mismatch",
'resolved adapter cwd "/tmp/agent-fallback"',
);
});
it("rejects a workspace-linked issue when no execution workspace was persisted", async () => {
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
persistedExecutionWorkspace: null,
}),
"missing_persisted_execution_workspace",
"requires a project execution workspace",
);
});
it("rejects a workspace-linked issue when no effective adapter cwd was resolved", async () => {
const input = buildWorkspaceValidationInput();
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
executionWorkspace: {
...input.executionWorkspace,
cwd: null,
},
}),
"missing_effective_cwd",
"no adapter cwd was resolved",
);
});
it("rejects a persisted execution workspace linked to a different project workspace", async () => {
const input = buildWorkspaceValidationInput();
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
persistedExecutionWorkspace: {
...input.persistedExecutionWorkspace!,
projectWorkspaceId: "workspace-other",
},
}),
"project_workspace_mismatch",
'expected project workspace "workspace-1"',
);
});
it("rejects a persisted execution workspace missing its project workspace id", async () => {
const input = buildWorkspaceValidationInput();
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
persistedExecutionWorkspace: {
...input.persistedExecutionWorkspace!,
projectWorkspaceId: null,
},
}),
"persisted_workspace_missing_project_workspace_id",
"has no project workspace id",
);
});
it("rejects a workspace-linked issue that would launch from the agent fallback cwd", async () => {
const input = buildWorkspaceValidationInput();
const fallbackCwd = resolveDefaultAgentWorkspaceDir("agent-1");
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
executionWorkspace: {
...input.executionWorkspace,
cwd: fallbackCwd,
},
persistedExecutionWorkspace: {
...input.persistedExecutionWorkspace!,
cwd: fallbackCwd,
},
}),
"fallback_agent_home_cwd",
"would launch from agent fallback cwd",
);
});
it("rejects a git worktree persisted workspace when cwd differs from providerRef", async () => {
const input = buildWorkspaceValidationInput();
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
executionWorkspace: {
...input.executionWorkspace,
strategy: "git_worktree",
cwd: "/tmp/worktree-current",
},
persistedExecutionWorkspace: {
...input.persistedExecutionWorkspace!,
strategyType: "git_worktree",
cwd: "/tmp/worktree-current",
providerRef: "/tmp/worktree-expected",
},
}),
"git_worktree_provider_ref_mismatch",
'expected git worktree "/tmp/worktree-expected"',
);
});
it("rejects a workspace-linked issue when adapter cwd has no git metadata", async () => {
const input = buildWorkspaceValidationInput();
const cwd = "/tmp/paperclip-workspace-without-git-metadata";
await expectWorkspaceValidationFailure(
buildWorkspaceValidationInput({
resolvedWorkspace: buildResolvedWorkspace({ cwd }),
executionWorkspace: {
...input.executionWorkspace,
baseCwd: cwd,
cwd,
},
persistedExecutionWorkspace: {
...input.persistedExecutionWorkspace!,
cwd,
},
}),
"missing_git_metadata",
"has no .git metadata",
);
});
it("does not apply the git-sensitive workspace guard to non-local execution targets", async () => {
const input = buildWorkspaceValidationInput();
await expect(
assertGitSensitiveAdapterWorkspaceValid(
buildWorkspaceValidationInput({
executionTarget: { kind: "cloud" },
executionWorkspace: {
...input.executionWorkspace,
cwd: "/tmp/agent-fallback",
},
}),
),
).resolves.toBeUndefined();
});
});
describe("stripHostWorkspaceProvisionForLowTrustSandbox", () => {
it("removes only the host-side provision command for sandbox-backed low-trust runs", () => {
const config = {
+271 -8
View File
@@ -240,6 +240,20 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_JITTER_RATIO = 0.25;
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure";
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry";
const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length;
const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
// Keep this in sync with local adapters that require a git workspace before launch.
const GIT_SENSITIVE_LOCAL_ADAPTER_TYPES = new Set([
"acpx_local",
"claude_local",
"codex_local",
"cursor",
"gemini_local",
"grok_local",
"hermes_local",
"opencode_local",
"pi_local",
]);
export const MAX_TURN_CONTINUATION_RETRY_REASON = "max_turns_continuation";
export const MAX_TURN_CONTINUATION_WAKE_REASON = "max_turns_continuation_retry";
const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
@@ -259,6 +273,17 @@ interface MaxTurnContinuationPolicy {
delayMs: number;
}
export class WorkspaceValidationFailure extends Error {
code = WORKSPACE_VALIDATION_FAILURE_CODE;
resultJson: Record<string, unknown>;
constructor(message: string, resultJson: Record<string, unknown>) {
super(message);
this.name = "WorkspaceValidationFailure";
this.resultJson = resultJson;
}
}
function resolveCodexTransientFallbackMode(attempt: number): CodexTransientFallbackMode {
if (attempt <= 1) return "same_session";
if (attempt === 2) return "safer_invocation";
@@ -891,6 +916,184 @@ async function ensureManagedProjectWorkspace(input: {
}
}
function isWorkspaceValidationFailure(error: unknown): error is WorkspaceValidationFailure {
return error instanceof WorkspaceValidationFailure;
}
function isWorkspaceValidationFailedRun(
run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode"> | null | undefined,
) {
return run?.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE;
}
async function hasGitMetadata(cwd: string | null | undefined) {
const normalized = readNonEmptyString(cwd);
if (!normalized) return false;
return fs
.lstat(path.resolve(normalized, ".git"))
.then((entry) => entry.isDirectory() || entry.isFile())
.catch(() => false);
}
function sameResolvedPath(left: string | null | undefined, right: string | null | undefined) {
const leftPath = readNonEmptyString(left);
const rightPath = readNonEmptyString(right);
if (!leftPath || !rightPath) return false;
return path.resolve(leftPath) === path.resolve(rightPath);
}
export async function assertGitSensitiveAdapterWorkspaceValid(input: {
adapterType: string;
agentId: string;
issue: {
id: string;
identifier: string | null;
projectId: string | null;
projectWorkspaceId: string | null;
} | null;
resolvedWorkspace: ResolvedWorkspaceForRun;
executionWorkspace: RealizedExecutionWorkspace;
persistedExecutionWorkspace: ExecutionWorkspace | null;
executionTarget: unknown;
environmentDriver?: string | null;
leaseMetadata?: unknown;
}) {
if (!GIT_SENSITIVE_LOCAL_ADAPTER_TYPES.has(input.adapterType)) return;
const executionTargetKind = readNonEmptyString((input.executionTarget as { kind?: unknown } | null)?.kind) ?? "local";
if (executionTargetKind !== "local") return;
const issue = input.issue;
if (!issue) return;
const environmentDriver = readNonEmptyString(input.environmentDriver) ?? "local";
const leaseMetadata = parseObject(input.leaseMetadata);
const leaseProviderMetadata = parseObject(leaseMetadata.providerMetadata);
const leaseRemoteCwd =
readNonEmptyString(leaseMetadata.remoteCwd) ??
readNonEmptyString(leaseProviderMetadata.remoteCwd);
const effectiveCwd = readNonEmptyString(input.executionWorkspace.cwd);
const persistedCwd = readNonEmptyString(input.persistedExecutionWorkspace?.cwd);
const agentFallbackCwd = resolveDefaultAgentWorkspaceDir(input.agentId);
const workspaceExpectation =
Boolean(issue.projectWorkspaceId) ||
Boolean(input.resolvedWorkspace.workspaceId) ||
input.executionWorkspace.strategy === "git_worktree";
const fail = (reason: string, message: string, extra: Record<string, unknown> = {}) => {
throw new WorkspaceValidationFailure(message, {
workspaceValidation: {
reason,
adapterType: input.adapterType,
issueId: issue.id,
issueIdentifier: issue.identifier,
issueProjectId: issue.projectId,
issueProjectWorkspaceId: issue.projectWorkspaceId,
resolvedWorkspaceSource: input.resolvedWorkspace.source,
resolvedProjectId: input.resolvedWorkspace.projectId,
resolvedProjectWorkspaceId: input.resolvedWorkspace.workspaceId,
resolvedWorkspaceCwd: input.resolvedWorkspace.cwd,
executionWorkspaceCwd: effectiveCwd,
executionWorkspaceStrategy: input.executionWorkspace.strategy,
executionWorkspaceProjectId: input.executionWorkspace.projectId,
executionWorkspaceProjectWorkspaceId: input.executionWorkspace.workspaceId,
persistedExecutionWorkspaceId: input.persistedExecutionWorkspace?.id ?? null,
persistedWorkspaceCwd: persistedCwd,
persistedWorkspaceStrategy: input.persistedExecutionWorkspace?.strategyType ?? null,
persistedProjectId: input.persistedExecutionWorkspace?.projectId ?? null,
persistedProjectWorkspaceId: input.persistedExecutionWorkspace?.projectWorkspaceId ?? null,
persistedProviderRef: input.persistedExecutionWorkspace?.providerRef ?? null,
...extra,
},
});
};
if (issue.projectWorkspaceId && !issue.projectId) {
fail(
"missing_project_id",
`Issue ${issue.identifier ?? issue.id} is linked to a project workspace but has no project id; refusing to launch ${input.adapterType} from fallback cwd.`,
);
}
if (!input.executionTarget && environmentDriver !== "local" && leaseRemoteCwd) return;
if (workspaceExpectation && !input.persistedExecutionWorkspace) {
fail(
"missing_persisted_execution_workspace",
`Issue ${issue.identifier ?? issue.id} requires a project execution workspace, but none was persisted before adapter launch.`,
);
}
if (workspaceExpectation && !effectiveCwd) {
fail(
"missing_effective_cwd",
`Issue ${issue.identifier ?? issue.id} expected a project workspace, but no adapter cwd was resolved before launch.`,
);
}
if (
input.persistedExecutionWorkspace &&
effectiveCwd &&
persistedCwd &&
!sameResolvedPath(effectiveCwd, persistedCwd)
) {
fail(
"persisted_cwd_mismatch",
`Issue ${issue.identifier ?? issue.id} resolved adapter cwd "${effectiveCwd}" but persisted execution workspace cwd is "${persistedCwd}".`,
);
}
const expectedProjectWorkspaceId = issue.projectWorkspaceId ?? input.resolvedWorkspace.workspaceId ?? null;
if (
expectedProjectWorkspaceId &&
input.persistedExecutionWorkspace &&
!input.persistedExecutionWorkspace.projectWorkspaceId
) {
fail(
"persisted_workspace_missing_project_workspace_id",
`Issue ${issue.identifier ?? issue.id} expected project workspace "${expectedProjectWorkspaceId}" but persisted execution workspace has no project workspace id.`,
);
}
if (
expectedProjectWorkspaceId &&
input.persistedExecutionWorkspace?.projectWorkspaceId &&
input.persistedExecutionWorkspace.projectWorkspaceId !== expectedProjectWorkspaceId
) {
fail(
"project_workspace_mismatch",
`Issue ${issue.identifier ?? issue.id} expected project workspace "${expectedProjectWorkspaceId}" but persisted execution workspace points at "${input.persistedExecutionWorkspace.projectWorkspaceId}".`,
);
}
if (workspaceExpectation && effectiveCwd && sameResolvedPath(effectiveCwd, agentFallbackCwd)) {
fail(
"fallback_agent_home_cwd",
`Issue ${issue.identifier ?? issue.id} expected a project workspace, but ${input.adapterType} would launch from agent fallback cwd "${effectiveCwd}".`,
);
}
if (
input.persistedExecutionWorkspace?.strategyType === "git_worktree" &&
input.persistedExecutionWorkspace.providerRef &&
effectiveCwd &&
!sameResolvedPath(effectiveCwd, input.persistedExecutionWorkspace.providerRef)
) {
fail(
"git_worktree_provider_ref_mismatch",
`Issue ${issue.identifier ?? issue.id} expected git worktree "${input.persistedExecutionWorkspace.providerRef}" but adapter cwd resolved to "${effectiveCwd}".`,
);
}
if (workspaceExpectation && effectiveCwd && !await hasGitMetadata(effectiveCwd)) {
fail(
"missing_git_metadata",
`Issue ${issue.identifier ?? issue.id} expected a git workspace for ${input.adapterType}, but "${effectiveCwd}" has no .git metadata.`,
);
}
}
const heartbeatRunProcessGroupIdColumn =
heartbeatRuns.processGroupId ?? sql<number | null>`NULL`.as("processGroupId");
@@ -8314,6 +8517,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const logEntry = formatRuntimeWorkspaceWarningLog(warning);
await onLog(logEntry.stream, logEntry.chunk);
}
await assertGitSensitiveAdapterWorkspaceValid({
adapterType: agent.adapterType,
agentId: agent.id,
issue: issueRef
? {
id: issueRef.id,
identifier: issueRef.identifier,
projectId: issueRef.projectId,
projectWorkspaceId: issueRef.projectWorkspaceId,
}
: null,
resolvedWorkspace,
executionWorkspace,
persistedExecutionWorkspace,
executionTarget,
environmentDriver: selectedEnvironment.driver,
leaseMetadata: activeEnvironmentLease.lease.metadata,
});
const adapterEnv = Object.fromEntries(
Object.entries(parseObject(resolvedConfig.env)).filter(
(entry): entry is [string, string] => typeof entry[0] === "string" && typeof entry[1] === "string",
@@ -8798,6 +9019,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
err instanceof Error ? err.message : "Unknown adapter failure",
await getCurrentUserRedactionOptions(),
);
const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err : null;
const failureErrorCode = workspaceValidationFailure?.code ?? "adapter_failed";
logger.error({ err, runId }, "heartbeat execution failed");
let logSummary: { bytes: number; sha256?: string; compressed: boolean } | null = null;
@@ -8818,11 +9041,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const failedRun = await setRunStatus(run.id, "failed", {
error: message,
errorCode: "adapter_failed",
errorCode: failureErrorCode,
finishedAt: new Date(),
resultJson: mergeRunStopMetadataForAgent(agent, "failed", {
errorCode: "adapter_failed",
errorCode: failureErrorCode,
errorMessage: message,
resultJson: workspaceValidationFailure?.resultJson ?? null,
}),
stdoutExcerpt,
stderrExcerpt,
@@ -8844,7 +9068,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
});
const livenessRun = await classifyAndPersistRunLiveness(failedRun) ?? failedRun;
await refreshContinuationSummaryForRun(livenessRun, agent);
await finalizeIssueCommentPolicy(livenessRun, agent);
if (!isWorkspaceValidationFailedRun(livenessRun)) {
await finalizeIssueCommentPolicy(livenessRun, agent);
}
await releaseIssueExecutionAndPromote(livenessRun);
await updateRuntimeState(agent, livenessRun, {
@@ -8907,7 +9133,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const failedAgent = setupFailureAgent ?? await getAgent(run.agentId).catch(() => null);
if (failedAgent) {
await refreshContinuationSummaryForRun(livenessRun, failedAgent).catch(() => undefined);
await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined);
if (!isWorkspaceValidationFailedRun(livenessRun)) {
await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch(() => undefined);
}
}
await releaseIssueExecutionAndPromote(livenessRun).catch(() => undefined);
}
@@ -8949,6 +9177,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
);
}
function buildWorkspaceValidationRecoveryComment(input: {
latestRun: Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode"> | null | undefined;
}) {
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
return (
"Paperclip stopped before launching the local adapter because the issue workspace failed validation. " +
`This prevents git-sensitive adapters from running in an unrelated fallback cwd.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the workspace link, cwd, or git checkout can be repaired before resuming."
);
}
async function releaseIssueExecutionAndPromote(run: typeof heartbeatRuns.$inferSelect) {
const runContext = parseObject(run.contextSnapshot);
const contextIssueId = readNonEmptyString(runContext.issueId);
@@ -9001,6 +9240,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
.where(eq(issues.id, issue.id));
}
if (
isWorkspaceValidationFailedRun(run) &&
(issue.status === "todo" || issue.status === "in_progress") &&
!issue.assigneeUserId &&
issue.assigneeAgentId === run.agentId
) {
return {
kind: "blocked" as const,
issue,
previousStatus: issue.status,
comment: buildWorkspaceValidationRecoveryComment({ latestRun: run }),
recoveryCause: WORKSPACE_VALIDATION_RECOVERY_CAUSE,
};
}
while (true) {
const deferred = await tx
.select()
@@ -9247,17 +9501,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const shouldBlockImmediately =
!recoveryAgentInvokable ||
!recoveryAgent ||
isWorkspaceValidationFailedRun(run) ||
didAutomaticRecoveryFail(run, issue.status === "todo" ? "assignment_recovery" : "issue_continuation_needed");
if (shouldBlockImmediately) {
const comment = buildImmediateExecutionPathRecoveryComment({
status: issue.status as "todo" | "in_progress",
latestRun: run,
});
const workspaceValidationFailure = isWorkspaceValidationFailedRun(run);
const comment = workspaceValidationFailure
? buildWorkspaceValidationRecoveryComment({ latestRun: run })
: buildImmediateExecutionPathRecoveryComment({
status: issue.status as "todo" | "in_progress",
latestRun: run,
});
return {
kind: "blocked" as const,
issue,
previousStatus: issue.status,
comment,
recoveryCause: workspaceValidationFailure ? WORKSPACE_VALIDATION_RECOVERY_CAUSE : undefined,
};
}
@@ -9340,6 +9599,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
previousStatus: promotionResult.previousStatus as "todo" | "in_progress",
latestRun: run,
comment: promotionResult.comment,
recoveryCause:
promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE
? WORKSPACE_VALIDATION_RECOVERY_CAUSE
: undefined,
});
return;
}
+10 -2
View File
@@ -5040,21 +5040,29 @@ export function issueService(db: Db) {
issueData.executionWorkspaceSettings !== undefined
? parseIssueExecutionWorkspaceSettings(issueData.executionWorkspaceSettings)
: parseIssueExecutionWorkspaceSettings(existing.executionWorkspaceSettings);
let validatedProjectWorkspace: { projectId: string } | null = null;
let validatedExecutionWorkspace: { projectId: string } | null = null;
if (!nextProjectId && nextProjectWorkspaceId) {
const workspace = await assertValidProjectWorkspace(existing.companyId, null, nextProjectWorkspaceId);
validatedProjectWorkspace = workspace;
nextProjectId = workspace.projectId;
patch.projectId = workspace.projectId;
}
if (!nextProjectId && nextExecutionWorkspaceId) {
const workspace = await assertValidExecutionWorkspace(existing.companyId, null, nextExecutionWorkspaceId);
validatedExecutionWorkspace = workspace;
nextProjectId = workspace.projectId;
patch.projectId = workspace.projectId;
}
if (nextProjectWorkspaceId) {
await assertValidProjectWorkspace(existing.companyId, nextProjectId, nextProjectWorkspaceId);
if (!validatedProjectWorkspace) {
await assertValidProjectWorkspace(existing.companyId, nextProjectId, nextProjectWorkspaceId);
}
}
if (nextExecutionWorkspaceId) {
await assertValidExecutionWorkspace(existing.companyId, nextProjectId, nextExecutionWorkspaceId);
if (!validatedExecutionWorkspace) {
await assertValidExecutionWorkspace(existing.companyId, nextProjectId, nextExecutionWorkspaceId);
}
}
applyStatusSideEffects(issueData.status, patch);
+22 -3
View File
@@ -102,7 +102,10 @@ type LatestIssueRun = Pick<
> | null;
type SuccessfulLatestIssueRun = NonNullable<LatestIssueRun> & { status: "succeeded" };
type StrandedRecoveryCause = "stranded_assigned_issue" | typeof SUCCESSFUL_RUN_MISSING_STATE_REASON;
type StrandedRecoveryCause =
| "stranded_assigned_issue"
| "workspace_validation_failed"
| typeof SUCCESSFUL_RUN_MISSING_STATE_REASON;
type SuccessfulRunHandoffRecoveryEvidence = {
sourceRunId: string | null;
@@ -2036,6 +2039,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
function strandedRecoveryActionKind(cause: StrandedRecoveryCause) {
return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
? "missing_disposition" as const
: cause === "workspace_validation_failed"
? "workspace_validation" as const
: "stranded_assigned_issue" as const;
}
@@ -2109,8 +2114,16 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
}),
nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
? "Choose and record a valid issue disposition without copying transcript content."
: recoveryCause === "workspace_validation_failed"
? "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution."
: "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.",
wakePolicy: ownerAgentId
wakePolicy: recoveryCause === "workspace_validation_failed"
? {
type: "manual_repair_required",
reason: "workspace_validation_failed",
ownerAgentId,
}
: ownerAgentId
? {
type: "wake_owner",
reason: "source_scoped_recovery_action",
@@ -2134,6 +2147,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
latestRun: LatestIssueRun;
recoveryCause: StrandedRecoveryCause;
}) {
if (input.recoveryCause === "workspace_validation_failed") return;
if (!input.action.ownerAgentId) return;
await deps.enqueueWakeup(input.action.ownerAgentId, {
source: "assignment",
@@ -2337,7 +2351,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
"- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
].join("\n");
if (recoveryAction.attemptCount === 1) {
const shouldPostEscalationComment =
recoveryAction.attemptCount === 1 ||
input.recoveryCause === "workspace_validation_failed";
if (shouldPostEscalationComment) {
const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``;
const hasEscalationComment = await db
@@ -2388,6 +2405,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
previousStatus: input.previousStatus,
source: input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
? "recovery.reconcile_successful_run_handoff_missing_state"
: input.recoveryCause === "workspace_validation_failed"
? "recovery.reconcile_workspace_validation_failed"
: "recovery.reconcile_stranded_assigned_issue",
recoveryCause: input.recoveryCause ?? "stranded_assigned_issue",
latestRunId: input.latestRun?.id ?? null,
+73 -3
View File
@@ -1,7 +1,7 @@
// @vitest-environment jsdom
import { act } from "react";
import { createRoot } from "react-dom/client";
import { flushSync } from "react-dom";
import type { AnchorHTMLAttributes, ReactElement, ReactNode } from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
@@ -27,6 +27,18 @@ vi.mock("../api/issues", () => ({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true;
function act<T>(callback: () => T): T {
let result: T | undefined;
flushSync(() => {
result = callback();
});
const maybePromise = result as unknown as PromiseLike<unknown>;
if (result && typeof maybePromise.then === "function") {
throw new TypeError("This test act shim only supports synchronous callbacks.");
}
return result as T;
}
let root: ReturnType<typeof createRoot> | null = null;
let container: HTMLDivElement | null = null;
let dateNowSpy: ReturnType<typeof vi.spyOn> | null = null;
@@ -158,9 +170,8 @@ describe("IssueBlockedNotice", () => {
expect(button).not.toBeNull();
expect(button!.textContent ?? "").toContain("Retry now");
await act(async () => {
act(() => {
button!.click();
await Promise.resolve();
});
await vi.waitFor(() => {
@@ -263,6 +274,65 @@ describe("IssueBlockedNotice", () => {
);
expect(indicator).not.toBeNull();
expect(indicator?.getAttribute("data-recovery-state")).toBe("needed");
expect(indicator?.getAttribute("data-recovery-kind")).toBe("missing_disposition");
expect(indicator?.textContent).toContain("Recovery needed");
});
it("labels a workspace_validation blocker recovery distinctly", () => {
const node = render(
<IssueBlockedNotice
issueStatus="blocked"
blockers={[
{
id: "blocker-2",
identifier: "PAP-409",
title: "Workspace cwd lost git context",
status: "blocked",
priority: "medium",
assigneeAgentId: null,
assigneeUserId: null,
activeRecoveryAction: {
id: "rec-2",
companyId: "co-1",
sourceIssueId: "blocker-2",
recoveryIssueId: null,
kind: "workspace_validation",
status: "active",
ownerType: "agent",
ownerAgentId: "agent-cto",
ownerUserId: null,
previousOwnerAgentId: null,
returnOwnerAgentId: null,
cause: "workspace_validation_failed",
fingerprint: "fp-2",
evidence: {
latestRunErrorCode: "workspace_validation_failed",
},
nextAction:
"Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.",
wakePolicy: { type: "wake_owner" },
monitorPolicy: null,
attemptCount: 1,
maxAttempts: 3,
timeoutAt: null,
lastAttemptAt: null,
outcome: null,
resolutionNote: null,
resolvedAt: null,
createdAt: "2026-05-01T00:00:00.000Z",
updatedAt: "2026-05-01T00:00:00.000Z",
},
},
]}
/>,
);
const indicator = node.querySelector(
'[data-testid="issue-blocked-notice-recovery-indicator"]',
);
expect(indicator).not.toBeNull();
expect(indicator?.getAttribute("data-recovery-state")).toBe("needed");
expect(indicator?.getAttribute("data-recovery-kind")).toBe("workspace_validation");
expect(indicator?.textContent).toContain("Workspace recovery needed");
});
});
+6 -3
View File
@@ -17,6 +17,7 @@ import { isAssignedBacklogBlocker } from "../lib/issue-blockers";
import {
deriveActiveRecoveryDisplayState,
RECOVERY_CHIP_DEFAULT_TONE,
recoveryChipLabel,
} from "../lib/recovery-display";
function BlockerRecoveryIndicator({ action }: { action: IssueRecoveryAction }) {
@@ -24,17 +25,19 @@ function BlockerRecoveryIndicator({ action }: { action: IssueRecoveryAction }) {
if (!state) return null;
const tone = RECOVERY_CHIP_DEFAULT_TONE[state];
const Icon = tone.icon;
const label = recoveryChipLabel(state, action.kind);
return (
<span
data-testid="issue-blocked-notice-recovery-indicator"
data-recovery-state={state}
data-recovery-kind={action.kind}
role="status"
aria-label={tone.label}
title={`${tone.label} — open the source issue to act.`}
aria-label={label}
title={`${label} — open the source issue to act.`}
className={`inline-flex shrink-0 items-center gap-0.5 rounded-full border px-1.5 py-0.5 text-[10px] font-medium ${tone.className}`}
>
<Icon className="h-2.5 w-2.5" aria-hidden />
{tone.label}
{label}
</span>
);
}
@@ -1,7 +1,7 @@
// @vitest-environment jsdom
import { act } from "react";
import { createRoot } from "react-dom/client";
import { flushSync } from "react-dom";
import type { AnchorHTMLAttributes, ReactElement } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Agent, IssueRecoveryAction } from "@paperclipai/shared";
@@ -16,6 +16,18 @@ vi.mock("@/lib/router", () => ({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true;
function act<T>(callback: () => T): T {
let result: T | undefined;
flushSync(() => {
result = callback();
});
const maybePromise = result as unknown as PromiseLike<unknown>;
if (result && typeof maybePromise.then === "function") {
throw new TypeError("This test act shim only supports synchronous callbacks.");
}
return result as T;
}
let root: ReturnType<typeof createRoot> | null = null;
let container: HTMLDivElement | null = null;
@@ -157,6 +169,33 @@ describe("IssueRecoveryActionCard", () => {
expect(node.textContent).toContain("OBSERVING ACTIVE RUN");
});
it("renders a workspace-specific label and headline for workspace_validation", () => {
const node = render(
<IssueRecoveryActionCard
action={buildAction({
kind: "workspace_validation",
cause: "workspace_validation_failed",
nextAction:
"Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.",
wakePolicy: { type: "manual_repair_required" },
evidence: {
recoveryCause: "workspace_validation_failed",
latestRunErrorCode: "workspace_validation_failed",
},
})}
/>,
);
const section = node.querySelector("section[aria-label]");
expect(section?.getAttribute("data-recovery-kind")).toBe("workspace_validation");
expect(node.textContent).toContain("Workspace Validation");
expect(node.textContent).not.toContain("workspace_validation\n");
expect(node.textContent).toContain(
"Paperclip stopped this run because the issue's git workspace could not be validated.",
);
expect(node.textContent).toContain("Repair the source issue workspace link");
expect(node.textContent).toContain("Manual repair required");
});
it("renders the resolved label and outcome when resolved", () => {
const node = render(
<IssueRecoveryActionCard action={buildAction({ status: "resolved", outcome: "restored", resolvedAt: "2026-05-09T19:35:00.000Z" })} />,
@@ -46,6 +46,7 @@ export interface IssueRecoveryActionCardProps {
const KIND_LABEL: Record<IssueRecoveryActionKind, string> = {
missing_disposition: "Missing Disposition",
stranded_assigned_issue: "Stranded Issue",
workspace_validation: "Workspace Validation",
active_run_watchdog: "Active Watchdog",
issue_graph_liveness: "Graph Liveness",
};
@@ -54,6 +55,8 @@ const KIND_HEADLINE: Record<IssueRecoveryActionKind, string> = {
missing_disposition: "This issue's run finished, but no next step was chosen.",
stranded_assigned_issue:
"Paperclip retried this issue's last run and it still has no live execution path.",
workspace_validation:
"Paperclip stopped this run because the issue's git workspace could not be validated.",
active_run_watchdog:
"The active run has been silent. Recovery is observing without interrupting it.",
issue_graph_liveness:
@@ -169,6 +172,7 @@ function readWakePolicySummary(action: IssueRecoveryAction): string | null {
if (type === "wake_owner") return "Corrective wake queued";
if (type === "board_escalation") return "Escalated to board";
if (type === "manual") return "Manual";
if (type === "manual_repair_required") return "Manual repair required";
if (type === "monitor") {
const interval = readEvidenceString(policy.intervalLabel);
return interval ? `Monitor scheduled · ${interval}` : "Monitor scheduled";
+10 -4
View File
@@ -8,7 +8,11 @@ import {
withIssueDetailHeaderSeed,
} from "../lib/issueDetailBreadcrumb";
import { cn } from "../lib/utils";
import { deriveActiveRecoveryDisplayState, RECOVERY_CHIP_DEFAULT_TONE } from "../lib/recovery-display";
import {
deriveActiveRecoveryDisplayState,
RECOVERY_CHIP_DEFAULT_TONE,
recoveryChipLabel,
} from "../lib/recovery-display";
import { StatusIcon } from "./StatusIcon";
import { productivityReviewTriggerLabel } from "./ProductivityReviewBadge";
import { hasAssignedBacklogBlocker } from "../lib/issue-blockers";
@@ -233,21 +237,23 @@ function renderRecoveryChip(action: IssueRecoveryAction, selected: boolean): Rea
if (!state) return null;
const tone = RECOVERY_CHIP_DEFAULT_TONE[state];
const Icon = tone.icon;
const label = recoveryChipLabel(state, action.kind);
return (
<span
data-testid="issue-row-recovery-indicator"
data-recovery-state={state}
data-recovery-kind={action.kind}
role="status"
aria-label={tone.label}
aria-label={label}
className={cn(
"ml-1.5 inline-flex shrink-0 items-center gap-0.5 rounded-full border px-2 py-0.5 text-[10px] font-medium",
tone.className,
selected ? "!border-muted-foreground !text-muted-foreground" : null,
)}
title={`${tone.label} — open the source issue to act.`}
title={`${label} — open the source issue to act.`}
>
<Icon className="h-2.5 w-2.5" aria-hidden />
{tone.label}
{label}
</span>
);
}
+49
View File
@@ -0,0 +1,49 @@
import { describe, expect, it } from "vitest";
import {
deriveActiveRecoveryDisplayState,
deriveRecoveryDisplayState,
recoveryChipLabel,
} from "./recovery-display";
describe("recoveryChipLabel", () => {
it("returns the workspace-specific label when kind is workspace_validation and state is needed", () => {
expect(recoveryChipLabel("needed", "workspace_validation")).toBe(
"Workspace recovery needed",
);
});
it("falls back to the generic label for other needed kinds", () => {
expect(recoveryChipLabel("needed", "missing_disposition")).toBe("Recovery needed");
expect(recoveryChipLabel("needed", "stranded_assigned_issue")).toBe("Recovery needed");
expect(recoveryChipLabel("needed", "issue_graph_liveness")).toBe("Recovery needed");
});
it("does not override the chip label for non-needed states", () => {
expect(recoveryChipLabel("in_progress", "workspace_validation")).toBe(
"Recovery in progress",
);
expect(recoveryChipLabel("escalated", "workspace_validation")).toBe(
"Recovery escalated",
);
expect(recoveryChipLabel("observe_only", "workspace_validation")).toBe(
"Observing active run",
);
});
});
describe("deriveRecoveryDisplayState", () => {
const base = {
status: "active" as const,
kind: "missing_disposition" as const,
outcome: null,
};
it("classifies workspace_validation active as needed", () => {
expect(deriveRecoveryDisplayState({ ...base, kind: "workspace_validation" })).toBe(
"needed",
);
expect(
deriveActiveRecoveryDisplayState({ ...base, kind: "workspace_validation" }),
).toBe("needed");
});
});
+11 -1
View File
@@ -1,4 +1,4 @@
import type { IssueRecoveryAction } from "@paperclipai/shared";
import type { IssueRecoveryAction, IssueRecoveryActionKind } from "@paperclipai/shared";
import { Eye, OctagonAlert, RefreshCw, TriangleAlert } from "lucide-react";
export type RecoveryDisplayState =
@@ -55,3 +55,13 @@ export function deriveActiveRecoveryDisplayState(
const state = deriveRecoveryDisplayState(action);
return state === "resolved" ? null : state;
}
export function recoveryChipLabel(
state: ActiveRecoveryDisplayState,
kind: IssueRecoveryActionKind,
): string {
if (kind === "workspace_validation" && state === "needed") {
return "Workspace recovery needed";
}
return RECOVERY_CHIP_DEFAULT_TONE[state].label;
}
@@ -129,6 +129,21 @@ function AllStatesPanel() {
nextAction: "Issue restored to a valid disposition.",
})}
/>
<CardPanel
caption="State 6 · Workspace validation (kind: workspace_validation)"
action={buildAction({
kind: "workspace_validation",
cause: "workspace_validation_failed",
fingerprint: "fp-workspace",
evidence: {
latestRunErrorCode: "workspace_validation_failed",
sourceRunId: "7accd7a4-c9ca-4db2-9233-3228a037cc09",
},
nextAction:
"Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.",
})}
canFalsePositive
/>
</div>
);
}
@@ -173,6 +188,20 @@ function BlockerNoticePanel() {
identifier: "PAP-9051",
title: "Bare blocker without recovery state",
}),
buildBlocker({
id: "blocker-5",
identifier: "PAP-10409",
title: "Phase 6 follow-up failed: git workspace lost",
status: "blocked",
activeRecoveryAction: buildAction({
kind: "workspace_validation",
cause: "workspace_validation_failed",
fingerprint: "fp-workspace-blocker",
evidence: { latestRunErrorCode: "workspace_validation_failed" },
nextAction:
"Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.",
}),
}),
]}
/>
</div>
@@ -322,6 +351,23 @@ function InboxRowPanel() {
activeRecoveryAction: buildAction({ status: "escalated" }),
}}
/>
<IssueRow
issue={{
...baseIssue,
id: "issue-recovery-workspace",
identifier: "PAP-10409",
title: "Phase 6 follow-up failed: git workspace lost",
status: "blocked",
activeRecoveryAction: buildAction({
kind: "workspace_validation",
cause: "workspace_validation_failed",
fingerprint: "fp-workspace-row",
evidence: { latestRunErrorCode: "workspace_validation_failed" },
nextAction:
"Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution.",
}),
}}
/>
</div>
);
}