diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0b5bba36e66..6e053b4606e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1379,6 +1379,14 @@ termination or saved files; the existing stop-and-save worker must establish bot before workspace selection can proceed. Both callers hold the task, coordinator and source-run locks throughout retirement and preservation registration. +A settled native bootstrap failure does not keep a workspace change blocked +solely because startup recorded `workspace_finalize=failed`. Admission requires +an explicit no-provider-work bootstrap receipt, a finished failed run, an unowned +terminal coordinator without a result, and no recorded provider events or process. +A restore failure, sync descriptor, copyback owner, unfinished operation, or pending +lease cleanup still blocks the change. This applies to next-admission selections +and legacy binding updates. Historical failure records remain unchanged. + When a same-run retry acquires a replacement plugin task lease, the executor reloads the prior task's durable Runner checkpoint before checking local lifecycle evidence. This includes retries after a provider turn omitted its diff --git a/server/src/__tests__/task-workspace-binding.test.ts b/server/src/__tests__/task-workspace-binding.test.ts index 6e4cf76cdf6..4350b661cb7 100644 --- a/server/src/__tests__/task-workspace-binding.test.ts +++ b/server/src/__tests__/task-workspace-binding.test.ts @@ -13,7 +13,7 @@ import request from "supertest"; import { taskWorkspaceRoutes } from "../routes/task-workspaces.js"; import { issueRoutes } from "../routes/issues.js"; import { errorHandler } from "../middleware/error-handler.js"; -import { agents, authUsers, companies, companyMemberships, completionContracts, createDb, environmentLeases, executionWorkspaceRepositories, executionWorkspaceRepositoryAccess, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, principalPermissionGrants, projectAccessMembers, projects, projectWorkspaces } from "@paperclipai/db"; +import { agents, authUsers, companies, companyMemberships, completionContracts, createDb, environmentLeases, executionWorkspaceRepositories, executionWorkspaceRepositoryAccess, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, principalPermissionGrants, projectAccessMembers, projects, projectWorkspaces } from "@paperclipai/db"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { authorizationService, canActorReadExecutionWorkspace } from "../services/authorization.js"; import { assertChatExecutionDefaultsAccess } from "../services/chat-execution-defaults.js"; @@ -418,6 +418,44 @@ const support = await getEmbeddedPostgresTestSupport(); } }); + it("allows a workspace change after a settled bootstrap failure with no provider work", async () => { + const f = await settledNativeTask("native_continuation_requires_reconciliation"); + await db.update(heartbeatRuns).set({ status: "failed", nativePhase: "observed", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + runnerProfileJson: { sessionCheckpoint: { providerSessionId: "inherited-session" } }, + }).where(eq(heartbeatRuns.id, f.run.id)); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: f.run.id, executionWorkspaceId: f.workspaceId, phase: "workspace_finalize", status: "failed", + metadata: { errorMessage: "runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate" } }); + await f.change(); + expect(await f.svc.inspectTaskWorkspace(f.companyId, f.issueId, f.actor)).toMatchObject({ + workspace: null, pendingSelection: null, + }); + expect((await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, f.run.id)))[0]) + .toMatchObject({ status: "failed" }); + }); + + it.each(["unknown startup", "provider started", "provider event", "process identity", "sync descriptor", + "copyback owner", "restore failure", "pending cleanup", "accepted result", "unfinished operation"] as const)( + "keeps a bootstrap failure with %s fenced", async (obligation) => { + const f = await settledNativeTask("native_continuation_requires_reconciliation", obligation === "accepted result"); + const recovery = { kind: "bootstrap", ...(obligation === "unknown startup" ? {} : { providerWorkStarted: obligation === "provider started" }) }; + await db.update(heartbeatRuns).set({ status: "failed", resultJson: { executionRecovery: recovery, + ...(obligation === "restore failure" ? { workspaceRestoreFailure: "restore_failed" } : {}) }, + ...(obligation === "process identity" ? { processPid: 123 } : {}), + runnerProfileJson: obligation === "sync descriptor" ? { nativeWorkspaceSync: { state: "prepared" } } + : obligation === "copyback owner" ? { nativeWorkspaceFinalizationOwner: { token: "owner" } } : {}, + }).where(eq(heartbeatRuns.id, f.run.id)); + if (obligation === "provider event") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, + agentId: f.run.agentId, runId: f.run.id, seq: 1, eventType: "turn.started", stream: "system", payload: {} }); + if (obligation === "pending cleanup") await db.update(environmentLeases).set({ status: "pending_cleanup" }) + .where(eq(environmentLeases.id, f.lease.id)); + await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: f.run.id, executionWorkspaceId: f.workspaceId, phase: "workspace_finalize", + status: obligation === "unfinished operation" ? "running" : "failed" }); + await expect(f.change()).rejects.toThrow(/workspace|Previous task run/); + }); + it.each(["warm release", "unsafe export omission"] as const)("accepts a settled %s receipt", async (receipt) => { const f = await settledNativeTask("native_retry_cancelled", receipt === "unsafe export omission"); if (receipt === "warm release") await db.update(environmentLeases).set({ releasedAt: null, status: "retained", leasePolicy: "reuse_by_environment", cleanupStatus: "success" }).where(eq(environmentLeases.id, f.lease.id)); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index ef4c2da940f..4b716916638 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -31,6 +31,8 @@ import { taskWorkspaceSelectionSchema, type TaskWorkspaceSelection, type TaskWor import { authorizationService, executionWorkspaceReadSqlCondition, projectReadSqlCondition, issueReadSqlCondition, type AuthorizationActor } from "./authorization.js"; import type { SQL } from "drizzle-orm"; import { createWorkspaceGitInspectionCache } from "./workspace-git-inspection-cache.js"; +import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; +import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { notifyTerminalWorkspaceWork, TERMINAL_WORKSPACE_RETRY_MS } from "./terminal-workspace-work.js"; import { deleteWorkspaceCompactionTrash, @@ -1401,9 +1403,22 @@ async function hasPendingTaskWorkspaceRecovery(tx: DbTransaction, input: { for (const operation of operations) { if (operation.runId && !latest.has(operation.runId)) latest.set(operation.runId, operation.status); } + const bootstrapRunIds = rows.filter(({ finalization, run }) => { + const recovery = parseObject(run.resultJson?.executionRecovery); + return run.runtimeMode === "native" && run.status === "failed" && run.finishedAt !== null + && finalization.phase === "terminal_failure" && finalization.resultId === null + && run.processPid === null && run.processGroupId === null + && recovery.kind === "bootstrap" && recovery.providerWorkStarted === false + && !hasWorkspaceRestoreFailure(run.resultJson) + && run.runnerProfileJson?.nativeWorkspaceSync == null + && run.runnerProfileJson?.nativeWorkspaceFinalizationOwner == null + && latest.get(run.id) === "failed"; + }).map(({ run }) => run.id); + const providerEvidence = await nativeSessionProviderEvidence(tx, bootstrapRunIds); + const noWorkFailures = new Set(bootstrapRunIds.filter(id => !providerEvidence.has(id))); return rows.some(({ finalization, run }) => { const status = latest.get(run.id); - if (status !== undefined) return status !== "succeeded"; + if (status !== undefined) return status !== "succeeded" && !noWorkFailures.has(run.id); return finalization.phase === "terminal_failure" && finalization.resultId !== null; }); }