diff --git a/server/src/services/__tests__/workspace-recovery-fixture.ts b/server/src/services/__tests__/workspace-recovery-fixture.ts index 901413e103a..da034421bfb 100644 --- a/server/src/services/__tests__/workspace-recovery-fixture.ts +++ b/server/src/services/__tests__/workspace-recovery-fixture.ts @@ -115,7 +115,8 @@ export function useWorkspaceRecoveryFixture(executionTargets: Map Promise) { - const prior = await seed("native", true, "keep_running", true), source = await run(prior.runId); +export async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof environmentLeases.$inferSelect) => Promise, + options: { firstOwnerInSharedWorkspace?: boolean } = {}) { + const sharedWorkspaceId = options.firstOwnerInSharedWorkspace ? randomUUID() : undefined; + const prior = await seed("native", true, "keep_running", true, false, sharedWorkspaceId), source = await run(prior.runId); + const descriptorWorkspaceId = sharedWorkspaceId ?? prior.issueId; + let executionWorkspaceId: string | null = null, taskId: string = prior.issueId; + if (sharedWorkspaceId) { + const [workspace] = await db.insert(executionWorkspaces).values({ id: sharedWorkspaceId, companyId: prior.companyId, + mode: "isolated_workspace", strategyType: "test", name: "Shared task workspace" }).returning(); + const [later] = await db.insert(issues).values({ companyId: prior.companyId, assigneeAgentId: source.agentId, + status: "blocked", title: "Later task in the same workspace" }).returning(); + executionWorkspaceId = workspace.id; taskId = later.id; + await db.update(environmentLeases).set({ executionWorkspaceId }).where(eq(environmentLeases.id, prior.lease.id)); + } let previousTarget = targets.get(prior.lease.id)!, previousLeaseId = prior.lease.id; let previousRunId: string = prior.runId; let lease = prior.lease, current = source; @@ -409,17 +422,17 @@ export async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof en await environmentService(db).releaseLease(previousLeaseId, "released", { cleanupStatus: "success" }); const [released] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, previousLeaseId)); [current] = await db.insert(heartbeatRuns).values({ companyId: prior.companyId, agentId: source.agentId, - issueId: prior.issueId, nativeIssueId: prior.issueId, runtimeMode: "native", status: "cancelled", finishedAt: new Date(), - contextSnapshot: { issueId: prior.issueId } }).returning(); - await db.insert(nativeRunFinalizations).values({ companyId: prior.companyId, issueId: prior.issueId, runId: current.id, phase: "terminal_failure" }); + issueId: taskId, nativeIssueId: taskId, runtimeMode: "native", status: "cancelled", finishedAt: new Date(), + contextSnapshot: { issueId: taskId } }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId: prior.companyId, issueId: taskId, runId: current.id, phase: "terminal_failure" }); const acquired = await environmentService(db).acquireLease({ companyId: prior.companyId, environmentId: prior.lease.environmentId!, - issueId: prior.issueId, heartbeatRunId: current.id, provider: "daytona", providerLeaseId: prior.providerLeaseId, - leasePolicy: "reuse_by_environment", replacesReusableLeaseId: previousLeaseId, + issueId: taskId, heartbeatRunId: current.id, provider: "daytona", providerLeaseId: prior.providerLeaseId, + executionWorkspaceId, leasePolicy: "reuse_by_environment", replacesReusableLeaseId: previousLeaseId, metadata: { ...released.metadata, sandboxLeaseAcquisition: { outcome: "resumed" } } }); [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, acquired.id)); const target = { ...previousTarget, leaseId: lease.id, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: prior.providerLeaseId } }; targets.set(lease.id, target); - expect(await prepareNativeWorkspaceSync({ db, companyId: prior.companyId, workspaceId: prior.issueId, + expect(await prepareNativeWorkspaceSync({ db, companyId: prior.companyId, workspaceId: descriptorWorkspaceId, workspaceLocalDir: prior.host, runId: current.id, lease: lease as EnvironmentLease, target, resourceDisposition: "keep_running" })).not.toBeNull(); leaseIds.push(lease.id); previousRunId = current.id; previousLeaseId = lease.id; previousTarget = target; } @@ -434,8 +447,8 @@ export async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof en expect(stopping).not.toBeNull(); expect(await settleStopOnlyCleanup(db, stopping!, { attemptId: String(stopping!.metadata!.pendingCleanupAttemptId), receipt: { providerLeaseId: prior.providerLeaseId, state: "stopped" } })).not.toBeNull(); - return { ...prior, priorRunId: prior.runId, priorLeaseId: prior.lease.id, leaseIds, runId: current.id, lease, actionId: action.id, - input: { ...prior.input, runId: current.id, actionId: action.id } }; + return { ...prior, issueId: taskId, priorRunId: prior.runId, priorLeaseId: prior.lease.id, firstOwnerIssueId: prior.issueId, executionWorkspaceId, + leaseIds, runId: current.id, lease, actionId: action.id, input: { ...prior.input, issueId: taskId, runId: current.id, actionId: action.id } }; } export async function makeRetryDue(actionId: string) { diff --git a/server/src/services/environment-lease-handoff.ts b/server/src/services/environment-lease-handoff.ts index 41df34ec05b..37b2ed148c4 100644 --- a/server/src/services/environment-lease-handoff.ts +++ b/server/src/services/environment-lease-handoff.ts @@ -61,7 +61,8 @@ export function retiredReusableLeasePredecessors(current: Lease, history: Lease[ (outgoing as Record)[key] !== value) || !previous.environmentId || !previous.issueId || typeof previous.metadata?.agentId !== "string" || previous.companyId !== current.companyId || previous.environmentId !== current.environmentId || - previous.executionWorkspaceId !== current.executionWorkspaceId || previous.issueId !== current.issueId || + previous.executionWorkspaceId !== current.executionWorkspaceId || + (current.executionWorkspaceId === null && previous.issueId !== current.issueId) || previous.metadata.agentId !== current.metadata?.agentId || previous.provider !== current.provider || previous.providerLeaseId !== current.providerLeaseId || previous.releasedAt?.getTime() !== successor.acquiredAt.getTime() || diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 7ea79bd92b0..78d2c443da1 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -412,6 +412,7 @@ export async function admitExplicitNativeContinuation(input: { ownerHold: sql`${issueRecoveryActions.evidence}#>>'{workspaceOwnerRecovery,state}' = 'operator_required' and ${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`, recoveryState: sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}'`, + retryClass: sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,failure,retryClass}'`, }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), inArray(issueRecoveryActions.status, ["active", "escalated"]), @@ -423,7 +424,9 @@ export async function admitExplicitNativeContinuation(input: { // starts one new recovery generation (task-retry), so name that action // instead of an endless finalizing wait (RC-50b, s14). if (live.length && live.every(row => row.recoveryState === "exhausted")) - return blocked("saved_work_recovery_failed", "Saving the agent's files failed after automatic retries. Retry to try again."); + return live.some(row => row.retryClass === "service_required") + ? blocked("saved_work_recovery_refused", "Paperclip refused to recover the saved files because a safety check on them cannot pass. Retry runs the same check again.") + : blocked("saved_work_recovery_failed", "Saving the agent's files failed. Retry to try again."); return blocked("finalizing_saved_result", "Paperclip is finishing the saved result."); } } diff --git a/server/src/services/system-recovery-state.test.ts b/server/src/services/system-recovery-state.test.ts index e04ba8ec10a..0a99d42720b 100644 --- a/server/src/services/system-recovery-state.test.ts +++ b/server/src/services/system-recovery-state.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { createSystemRecovery, failSystemRecovery, readSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "./system-recovery-state.js"; +import { createSystemRecovery, failSystemRecovery, isDeterministicRecoveryRefusal, readSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "./system-recovery-state.js"; import { exhaustedSystemRecoveryAction } from "./system-recovery.js"; import { workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; @@ -59,17 +59,36 @@ describe("durable system recovery budget", () => { message: "Automatic recovery of saved work stopped after 1 attempts (last failure: allocation_ownership_ambiguous). " + "Retry recovery to export and merge from the retained sandbox again; no provider turn is submitted. " + "If Retry keeps failing, inspect the run and its environment." } }, runId); - expect(display?.message).toContain("Retry to try again"); + expect(display?.message).toContain("Paperclip refused to recover the saved files"); expect(display?.message).not.toMatch(manual); }); + it("states a deterministic refusal plainly and never claims retries that did not happen", () => { + const refused = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required"), { now: 0 }); + expect(refused).toMatchObject({ state: "exhausted", attempts: 1, retriesUsed: 0 }); + expect(isDeterministicRecoveryRefusal(refused)).toBe(true); + const text = exhaustedSystemRecoveryAction(refused); + expect(text).toContain("Paperclip refused to recover the saved files"); + expect(text).toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files (allocation_ownership_ambiguous)"); + expect(text).toContain("Retry runs the check again and succeeds only after Paperclip is updated"); + expect(text).not.toMatch(/after automatic retries|stopped after|Retry to try again/); + + const once = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, + new SystemRecoveryFailure("historical_source_missing", "use_alternative_strategy"), { now: 0 }); + expect(isDeterministicRecoveryRefusal(once)).toBe(false); + expect(exhaustedSystemRecoveryAction(once)).toBe("Automatic recovery of saved work failed (last failure: historical_source_missing). " + + "Retry to try again; the agent will not repeat its work."); + expect(exhaustedSystemRecoveryAction({ ...once, attempts: 6 })).toContain("stopped after 6 attempts"); + }); + it("names the last failure and its cause in the board handoff", () => { const state = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; const withCause = failSystemRecovery(state, new SystemRecoveryFailure("provider_unavailable", "retry_transient"), { now: 0, cause: { label: "sandbox_not_running", code: null, status: null } }); expect(exhaustedSystemRecoveryAction(withCause)).toContain("(last failure: provider_unavailable, cause: sandbox_not_running)"); const withoutCause = failSystemRecovery(state, new SystemRecoveryFailure("archive_unsafe", "service_required"), { now: 0 }); - expect(exhaustedSystemRecoveryAction(withoutCause)).toContain("(last failure: archive_unsafe)"); + expect(exhaustedSystemRecoveryAction(withoutCause)).toContain("(archive_unsafe)"); }); it("reconciles a recorded attempt after restart without spending another retry", () => { const running = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index 703bf0fa9b6..a70b6ed6c0c 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -135,7 +135,26 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState): strin return "Recovery of saved work is waiting for the previous server's file copy to stop. " + "Recovery resumes once Paperclip can confirm that it stopped. No provider turn is submitted."; const cause = state.failure?.cause ? `, cause: ${state.failure.cause.label}` : ""; + if (isDeterministicRecoveryRefusal(state)) { + const code = state.failure!.code; + return `Paperclip refused to recover the saved files: ${recoveryRefusalReasons[code] ?? "a safety check on the saved files did not pass"} ` + + `(${code}${cause}). Automatic recovery does not repeat this check, because the same saved state gives the same result. ` + + "Retry runs the check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work."; + } const reason = state.failure?.code ? ` (last failure: ${state.failure.code}${cause})` : ""; - return `Automatic recovery of saved work stopped after ${state.attempts} attempts${reason}. ` + const outcome = state.attempts > 1 ? `stopped after ${state.attempts} attempts` : "failed"; + return `Automatic recovery of saved work ${outcome}${reason}. ` + "Retry to try again; the agent will not repeat its work."; } + +export function isDeterministicRecoveryRefusal(state: Pick | null | undefined) { + return state?.state === "exhausted" && state.failure?.retryClass === "service_required"; +} + +const recoveryRefusalReasons: Partial> = { + allocation_ownership_ambiguous: "Paperclip cannot prove which runs used the sandbox that holds the saved files", + source_binding_invalid: "the recorded source of the saved files does not match the run that saved them", + snapshot_verification_failed: "the saved snapshot of the files did not pass verification", + archive_unsafe: "the saved file archive contains entries that are not safe to apply", + source_unavailable: "the saved files are no longer available where the run recorded them", +}; diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index f4b9d443c51..fab5116159c 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -123,7 +123,9 @@ const retryWaitMessages: Record = { finalizing_saved_result: "Paperclip is finishing the saved result before Retry can continue.", // Another click cannot release a copyback owner that stopped without reporting. // Retry starts a new recovery generation; the agent does not repeat its work. - saved_work_recovery_failed: "Saving the agent's files failed after automatic retries. Retry to try again; the agent will not repeat its work.", + saved_work_recovery_failed: "Saving the agent's files failed. Retry to try again; the agent will not repeat its work.", + saved_work_recovery_refused: "Paperclip refused to recover the saved files because a safety check on them cannot pass. " + + "Retry runs the same check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work.", workspace_owner_stop_required: "Retry is waiting until Paperclip confirms that the previous server stopped saving files. The agent's result and your Retry are saved.", cleanup_quarantined: "Retry cannot continue this attempt. Send a new message after the previous execution has stopped.", cancelled_by_operator: "This attempt was stopped. Send a new message to continue from saved work.", diff --git a/server/src/services/workspace-recovery-preparation-gates.test.ts b/server/src/services/workspace-recovery-preparation-gates.test.ts index 749288c8a57..e569bd4f3d4 100644 --- a/server/src/services/workspace-recovery-preparation-gates.test.ts +++ b/server/src/services/workspace-recovery-preparation-gates.test.ts @@ -416,6 +416,36 @@ describe("automatic retained workspace recovery", () => { .toEqual([{ id: settled.runId, previousRunId: f.runId }]); }, 60_000); + it("names a deterministic saved-files refusal behind Retry when no fallback can continue, and another Retry shows it again", async () => { + const f = await acceptedRemoteExport(); + const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Earlier task" }).returning(); + await db.insert(environmentLeases).values({ companyId: f.companyId, environmentId: lease.environmentId, issueId: other.id, + provider: "daytona", providerLeaseId: f.providerLeaseId, status: "expired", leasePolicy: "reuse_by_environment", + cleanupStatus: "success", releasedAt: new Date(), metadata: { remoteCwd: path.dirname(f.remote) } }); + await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, heartbeatRunId: f.runId, + issueId: f.issueId, phase: "remote_workspace_snapshot", status: "running", metadata: { boundary: "final" } }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.source.agentId, status: "running" }); + const app = boardRecoveryApp(); + for (const click of [1, 2]) { + const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`) + .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; + await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", attempts: 1, + failure: { code: "allocation_ownership_ambiguous", retryClass: "service_required" } }); + await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0], `click ${click}`) + .toMatchObject({ runId: null, payload: { executionWait: { reason: "saved_work_recovery_refused" } } }); + const shown = await taskInterruptionPresentation(db, issue); + expect(shown).toMatchObject({ state: "failed", canRetry: true, retryWait: { reason: "saved_work_recovery_refused" } }); + expect(shown?.retryWait?.message).toContain("Paperclip refused to recover the saved files"); + expect(shown?.retryWait?.message).not.toMatch(/after automatic retries/); + expect((await getExecutionBlocker(db, f.companyId, f.issueId))?.nextAction) + .toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files"); + } + }, 60_000); + it("preserves an explicitly requested superseded remote export without applying its result over newer work", async () => { const f = await acceptedRemoteExport(); const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.source.agentId, diff --git a/server/src/services/workspace-recovery-retry-copyback.test.ts b/server/src/services/workspace-recovery-retry-copyback.test.ts index cb0a5895304..cdc45a308ef 100644 --- a/server/src/services/workspace-recovery-retry-copyback.test.ts +++ b/server/src/services/workspace-recovery-retry-copyback.test.ts @@ -508,7 +508,7 @@ describe("automatic retained workspace recovery", () => { const stable = (row: typeof after) => ({ ...row, updatedAt: null, ownerType: null, nextAction: null, evidence: { ...row.evidence, workspaceRecovery: { ...(row.evidence.workspaceRecovery as object), message: null } } }); if (spent) { - expect(after).toMatchObject({ ownerType: "board", nextAction: expect.stringMatching(/Retry to try again/) }); + expect(after).toMatchObject({ ownerType: "board", nextAction: expect.stringMatching(/Paperclip refused to recover the saved files: .*Retry runs the check again/) }); expect(stable(after)).toEqual(stable(before)); } else expect(after).toEqual(before); expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].leasePolicy).toBe("reuse_by_environment"); diff --git a/server/src/services/workspace-recovery-source.ts b/server/src/services/workspace-recovery-source.ts index a27555e3607..04b4f04cc0e 100644 --- a/server/src/services/workspace-recovery-source.ts +++ b/server/src/services/workspace-recovery-source.ts @@ -190,9 +190,8 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec const priors = predecessorRuns.length ? await db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), inArray(heartbeatRuns.id, predecessorRuns))) : []; const priorOwners = predecessorRuns.length ? await db.select({ runId: nativeRunFinalizations.runId, - phase: nativeRunFinalizations.phase, leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, input.companyId), inArray(nativeRunFinalizations.runId, predecessorRuns), - eq(nativeRunFinalizations.issueId, input.issueId))) : []; + issueId: nativeRunFinalizations.issueId, phase: nativeRunFinalizations.phase, leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), inArray(nativeRunFinalizations.runId, predecessorRuns))) : []; for (const predecessor of predecessors) { const prior = priors.find(item => item.id === predecessor.heartbeatRunId); const priorReference = readNativeWorkspaceSyncReference(prior?.runnerProfileJson?.nativeWorkspaceSync); @@ -203,7 +202,8 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec : priorRemote?.state === "saved" && Boolean(priorRemote.snapshotId) && Boolean(remoteDescriptor) && priorRemote.epoch < remoteDescriptor!.epoch; const priorOwner = priorOwners.find(item => item.runId === predecessor.heartbeatRunId); - if (!prior || prior.runtimeMode !== "native" || prior.agentId !== run.agentId || prior.nativeIssueId !== input.issueId || + if (!prior || prior.runtimeMode !== "native" || prior.agentId !== run.agentId || + !predecessor.issueId || prior.nativeIssueId !== predecessor.issueId || priorOwner?.issueId !== predecessor.issueId || !terminalWorkspaceRunStates.includes(prior.status) || adapterExecutionControls.has(prior.id) || (prior.controllerLeaseExpiresAt && prior.controllerLeaseExpiresAt > new Date()) || !priorOwner || priorOwner.leaseOwner || !["committed", "terminal_failure", "applied"].includes(priorOwner.phase) || diff --git a/server/src/services/workspace-recovery-warm-handoff.test.ts b/server/src/services/workspace-recovery-warm-handoff.test.ts index 220ffa96037..a97e26a569f 100644 --- a/server/src/services/workspace-recovery-warm-handoff.test.ts +++ b/server/src/services/workspace-recovery-warm-handoff.test.ts @@ -186,6 +186,37 @@ describe("automatic retained workspace recovery", () => { } }, 30_000); + it.each([1, 2])("recovers saved work after %s warm handoffs that began with another task in the same workspace", async hops => { + const f = await warmHandoffSource(hops, undefined, { firstOwnerInSharedWorkspace: true }); + const [first] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.priorLeaseId)); + expect(first).toMatchObject({ issueId: f.firstOwnerIssueId, executionWorkspaceId: f.executionWorkspaceId, status: "expired" }); + expect(new Set([f.firstOwnerIssueId, f.issueId, f.executionWorkspaceId]).size).toBe(3); + await recoverRetainedWorkspace(f.input); + const state = readSystemRecovery((await action(f.actionId)).evidence)!; + expect(state.failure?.code ?? null).toBeNull(); + expect(state.state).toBe("complete"); + expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true); + expect(await readFile(path.join(f.host, "after-warm-stop.txt"), "utf8")).toBe("stopped turn work\n"); + }, 30_000); + + it.each(["lease_task", "unshared_workspace", "other_workspace"])( + "keeps a cross-task predecessor fenced unless its own records prove the shared workspace: %s", async fault => { + const f = await warmHandoffSource(1, undefined, { firstOwnerInSharedWorkspace: true }); + const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Unrelated task" }).returning(); + if (fault === "lease_task") await db.update(environmentLeases).set({ issueId: other.id }).where(eq(environmentLeases.id, f.priorLeaseId)); + if (fault === "unshared_workspace") await db.update(environmentLeases).set({ executionWorkspaceId: null }) + .where(eq(environmentLeases.id, f.priorLeaseId)); + if (fault === "other_workspace") { + const [otherWorkspace] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, mode: "isolated_workspace", + strategyType: "test", name: "Other workspace" }).returning(); + await db.update(environmentLeases).set({ issueId: f.issueId, executionWorkspaceId: otherWorkspace.id }) + .where(eq(environmentLeases.id, f.priorLeaseId)); + } + expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, actionId: f.actionId })) + .toMatchObject({ status: "blocked", failureCode: "allocation_ownership_ambiguous" }); + await expect(readFile(path.join(f.host, "after-warm-stop.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + }, 30_000); + it.each(["active", "active_run", "active_control", "foreign_company", "environment", "workspace", "task", "agent", "unfinalized", "owner", "missing_proof", "conflicting_proof", "dangling_proof", "extra_history"])( "keeps ambiguous warm source history fenced: %s", async fault => { const f = await warmHandoffSource();