From 54ac0032dcde8dcc9851d6cf66cc5c09c7f0dc30 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Thu, 12 Nov 2026 20:06:46 -0800 Subject: [PATCH] Verify plugin workspaces on run leases and release settled tasks Acquire the logical placement before verifying a retained checkout and reuse it through preparation. Persist definite refusal before release and preserve the existing workspace and session choices on fresh fallback and restart. Always invoke provider release after task settlement. Reuse saved stop proof during cleanup retries and preserve completed run outcomes. Add database regressions for verification placement, restart, refusal, transport failure, and release replay. Co-Authored-By: Paperclip --- doc/plugins/PLUGIN_RUNNER_EXECUTION.md | 10 +- .../plugin-runner-task.integration.test.ts | 2 +- .../plugin-warm-tasks.integration.test.ts | 34 +- .../plugin-workspaces.integration.test.ts | 32 +- .../services/environment-run-orchestrator.ts | 2 + server/src/services/environment-runtime.ts | 68 ++-- .../src/services/environment-task-runtime.ts | 11 +- .../services/fresh-workspace-continuation.ts | 21 +- server/src/services/heartbeat.ts | 334 ++++++++++-------- .../heartbeat/execution-configuration.ts | 33 +- server/src/services/heartbeat/recovery.ts | 3 +- .../heartbeat/workspace-preparation.test.ts | 102 +++++- .../heartbeat/workspace-preparation.ts | 137 +++++-- server/src/services/plugin-workspaces.ts | 12 +- 14 files changed, 539 insertions(+), 262 deletions(-) diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index 4ca856513cd..3d52d0e11c8 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -204,12 +204,16 @@ A storage provider can declare `workspaceProvider.workspaceVerificationVersion: Before creating a fresh workspace for an admitted Retry, Core may send the original task-owned workspace's `prepare` request with `verifyExisting: true`. Core sends verification through `workspaceOperation` with a 10-second -deadline and a 12-second RPC timeout. +deadline and a 12-second RPC timeout. Drivers with task placement receive the +selected run's `taskLease` before Core describes the Runner task. Use that lease +to select the same execution placement for verification and later preparation. The provider must verify a completed checkout without creating or fetching one and return `prepared` with `created: false`. Core checks prior writer settlement and continues normal preparation on the retained workspace after -verification. A missing or unconfirmed checkout leaves the fresh retry -fallback in place. Local workspaces use the same ownership and writer +verification. Return the workspace failure reason `binding_mismatch` for a +definitively missing or invalid checkout. Core can then release the verification +lease and use the authorized fresh retry fallback. Transport failures and +timeouts keep the current checkout decision retryable. Local workspaces use the same ownership and writer checks with Core's existing local Git verification. A storage provider can declare `workspaceProvider.finalizationVersion: 1`. diff --git a/server/src/__tests__/plugin-runner-task.integration.test.ts b/server/src/__tests__/plugin-runner-task.integration.test.ts index fa7cebf6cdd..403eba17f2e 100644 --- a/server/src/__tests__/plugin-runner-task.integration.test.ts +++ b/server/src/__tests__/plugin-runner-task.integration.test.ts @@ -416,7 +416,7 @@ databaseSuite("Core plugin Runner task lifecycle", () => { return; } const next = await acquire; - expect(operations).toEqual(["stop", "status", "complete", "environmentAcquireLease"]); + expect(operations).toEqual(["stop", "status", "complete", "environmentReleaseLease", "environmentAcquireLease"]); expect(next.lease.id).not.toBe(oldLeaseId); await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).resolves.toBe("old-task"); await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", projectIds: [], bootstrapTicket: "fixture", runner } })).rejects.toThrow("identity mismatch"); diff --git a/server/src/__tests__/plugin-warm-tasks.integration.test.ts b/server/src/__tests__/plugin-warm-tasks.integration.test.ts index 88d4d8605ff..e2da8aa0d80 100644 --- a/server/src/__tests__/plugin-warm-tasks.integration.test.ts +++ b/server/src/__tests__/plugin-warm-tasks.integration.test.ts @@ -23,6 +23,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport const companyId=randomUUID(),agentId=randomUUID(),issueId=randomUUID(),environmentId=randomUUID(),pluginId=randomUUID(); let runId=randomUUID(),allocation=0,stopped=false,outage=false,releaseFails=false,unavailable=false; const calls:string[]=[],dispatched:Array<[string,string]>=[],failures=new Set(); + const releaseMetadata: unknown[] = []; let projectIds=[...initialProjects],providerExpiry=initialExpiry; const taskExpiries=new Map(),acquireRequests:any[]=[]; let resumeGate:Promise|undefined,releaseResume:(()=>void)|undefined,resumes=0; @@ -47,7 +48,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport } if(method==="environmentResumeLease") {if(outage) throw Error("unreachable"); if(resumeGate){if(++resumes===2)releaseResume!();await resumeGate;}return {providerLeaseId:stopped||unavailable?null:p.providerLeaseId,expiresAt:taskExpiries.get(p.providerLeaseId),metadata:{runId:p.runId}};} - if(method==="environmentReleaseLease") {if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};} + if(method==="environmentReleaseLease") {releaseMetadata.push(p.leaseMetadata);if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};} if(p.operation.kind==="status") return {kind:"status",taskId:p.taskId,phase:stopped?"completed":"running",executionStopped:stopped,sessionStateAvailable:stopped,preparationStopped:stopped}; if(p.operation.kind==="read_state") return {kind:"state",taskId:p.taskId,state:{...p.operation.identity,lifecycle:"ready"}}; if(p.operation.kind==="stop") stopped=true; @@ -66,11 +67,11 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport launch:{runnerVersion:"fixture",runnerDigest:`sha256:${"a".repeat(64)}`,maxOutboxBytes:262144,p0ReserveBytes:65536,lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}}}}); await runtime().releaseRunLeases(runId,"released",undefined,"keep_running"); await db.update(heartbeatRuns).set({status:"succeeded"}).where(eq(heartbeatRuns.id,runId)); - return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests, + return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests,releaseMetadata, fail(kind:string){failures.add(kind);},unavailable(){unavailable=true;}, setExpiry(next?:string){providerExpiry=next;}, async nextRun(){runId=randomUUID();await db.insert(heartbeatRuns).values({id:runId,companyId,agentId,nativeIssueId:issueId,invocationSource:"manual",status:"running"});return runId;}, - setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},outage(){outage=true;}, + setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},restoreRelease(){releaseFails=false;},outage(){outage=true;}, raceResume(){resumeGate=new Promise(resolve=>{releaseResume=resolve;});}}; } it("hands a retained process to a new run after controller reconstruction",async()=>{ @@ -118,7 +119,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport if(prepared) await db.update(environmentLeases).set({metadata:sql`${environmentLeases.metadata} || ${JSON.stringify({runnerTaskStartup:{contextId:"settled-startup",deadlineAtMs:1,runId:f.identity.runId}})}::jsonb`}).where(eq(environmentLeases.id,f.first.lease.id)); const calls=f.calls.length,dispatched=f.dispatched.length; const next=await f.acquire(); - expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentAcquireLease"]); + expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentReleaseLease","environmentAcquireLease"]); expect(f.dispatched.slice(dispatched)).toEqual([["status",f.first.lease.providerLeaseId],["complete",f.first.lease.providerLeaseId]]); expect(next.lease.providerLeaseId).not.toBe(f.first.lease.providerLeaseId); const [old]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,f.first.lease.id)); @@ -129,7 +130,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport const f=await fixture();f.unavailable();await f.nextRun(); const calls=f.calls.length; await f.acquire(); - expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentAcquireLease"]); + expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentReleaseLease","environmentAcquireLease"]); }); it.each([["status",false],["complete",false],["stop",true]] as const)("keeps a stopped retained task for cleanup retry when %s fails",async(kind,running)=>{ const f=await fixture();if(running) f.unavailable(); else f.stopped();await f.nextRun(); @@ -143,6 +144,29 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport await expect(f.runtime().retryPendingSandboxTeardown({environment:null,lease:pending})).resolves.toEqual({providerLeaseId:f.first.lease.providerLeaseId,state:"stopped"}); expect(f.calls.slice(calls)).toEqual(["stop","status","complete","environmentReleaseLease"]); }); + it("retries lease release after completed task settlement without repeating settlement", async () => { + const f = await fixture(); + f.stopped(); + f.failRelease(); + await f.nextRun(); + const before = f.calls.length; + await expect(f.acquire()).rejects.toThrow("release unavailable"); + expect(f.calls.slice(before)).toEqual(["environmentResumeLease", "status", "complete", "environmentReleaseLease"]); + const pending = (await environmentService(db).getLeaseById(f.first.lease.id))!; + expect(pending).toMatchObject({ + status: "pending_cleanup", cleanupStatus: "failed", + metadata: { runnerTaskDispatched: false, runnerTaskExecutionStopped: true }, + }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.identity.runId)); + expect(run.status).toBe("succeeded"); + f.restoreRelease(); + const retry = f.calls.length; + await expect(f.runtime().retryPendingSandboxTeardown({ environment: null, lease: pending })) + .resolves.toEqual({ providerLeaseId: f.first.lease.providerLeaseId, state: "stopped" }); + expect(f.calls.slice(retry)).toEqual(["environmentReleaseLease"]); + expect(f.releaseMetadata).toHaveLength(2); + for (const metadata of f.releaseMetadata) expect(metadata).toMatchObject({ runnerTaskDispatched: false, runnerTaskExecutionStopped: true }); + }); it("does not allocate a replacement when the retained computer cannot be inspected",async()=>{ const f=await fixture();f.outage();await f.nextRun(); await expect(f.acquire()).rejects.toThrow("unreachable"); diff --git a/server/src/__tests__/plugin-workspaces.integration.test.ts b/server/src/__tests__/plugin-workspaces.integration.test.ts index 7c21470b53d..a412bc0196f 100644 --- a/server/src/__tests__/plugin-workspaces.integration.test.ts +++ b/server/src/__tests__/plugin-workspaces.integration.test.ts @@ -1,3 +1,4 @@ +import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk"; import { isPluginWorkspaceReplay } from "../services/plugin-workspace-replay-context.js"; import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js"; import { randomUUID } from "node:crypto"; @@ -51,7 +52,7 @@ const support = await getEmbeddedPostgresTestSupport(); }); const fixture = (companyId?: string) => createPluginWorkspaceFixture(db, companyId); - it.each(["ready", "lost_reply", "unconfirmed", "live_writer", "unsupported"] as const)( + it.each(["ready", "lost_reply", "unconfirmed", "transport", "worker_lookup", "live_writer", "older_retained", "unsupported"] as const)( "verifies the same checkout before choosing a fresh retry (%s)", async (state) => { const f = await fixture(); const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId)); @@ -73,6 +74,15 @@ const support = await getEmbeddedPostgresTestSupport(); status: state === "live_writer" ? "active" : "released", releasedAt: state === "live_writer" ? null : new Date(), }); + if (state === "older_retained") { + const [older] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId, + issueId: f.issueId, status: "succeeded", finishedAt: new Date() }).returning(); + await db.insert(environmentLeases).values({ + companyId: f.companyId, issueId: f.issueId, heartbeatRunId: older.id, + provider: "plugin:test:tasks", executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_environment", + status: "retained", releasedAt: null, + }); + } const retryIntentId = randomUUID(); const [retry] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId, issueId: f.issueId, status: "running", contextSnapshot: { taskRetryRequestId: retryIntentId }, @@ -80,24 +90,34 @@ const support = await getEmbeddedPostgresTestSupport(); const fresh = { mode: "fresh_isolated_workspace", retryIntentId, sourceRunId: f.runId, retainedSourceIds: [] } as const; const call = vi.fn(async (...args) => { const result = await f.workers.call(...args) as { kind: string }; - if (state === "unconfirmed") throw new Error("checkout incomplete"); + if (state === "transport") throw new Error("connection closed"); + if (state === "unconfirmed") throw new JsonRpcCallError({ + code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: "checkout incomplete", + data: workspaceOperationErrorData(new PluginWorkspaceOperationError("binding_mismatch")), + }); return { ...result, created: false }; }); const workers: WorkspaceWorkers = { getWorker: (pluginId) => { + if (state === "worker_lookup") throw new Error("worker lookup timed out"); const worker = f.workers.getWorker(pluginId); return worker && { ...worker, supportedMethods: ["workspaceOperation", "workspacePrepare"] }; }, call, }; const verificationStarted = Date.now(); - const result = await verifyRetryWorkspace(db, workers, retry, f.issueId, null, - { ...fresh, retainedSourceIds: [] }); + const result = verifyRetryWorkspace(db, workers, retry, f.issueId, null, fresh); + if (state === "transport" || state === "worker_lookup") { + if (state === "transport") await expect(result).rejects.toMatchObject({ kind: "transport", retryable: true }); + else await expect(result).rejects.toThrow("worker lookup timed out"); + return; + } const verified = state === "ready" || state === "lost_reply"; - expect(result).toEqual(verified + expect(await result).toEqual(verified ? { mode: "reuse_issue_workspace", retryIntentId, sourceRunId: f.runId, workspaceId, cwd: null } : fresh); - expect(f.calls.slice(1)).toHaveLength(state === "live_writer" || state === "unsupported" ? 0 : 1); + expect(f.calls.slice(1)).toHaveLength(["live_writer", "older_retained", "unsupported"].includes(state) ? 0 : 1); if (f.calls.length > 1) { expect(call).toHaveBeenCalledExactlyOnceWith(f.pluginId, "workspaceOperation", expect.objectContaining({ executionWorkspaceId: workspaceId, runId: retry.id, operation: expect.objectContaining({ verifyExisting: true }), diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 77abc489223..918385e39f3 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -317,6 +317,7 @@ export function environmentRunOrchestrator( | { kind: "plugin_task"; leaseId: string; preparation?: true }; /** A new controller must stop the previous plugin task before allocating a replacement. */ recoverPluginTask?: boolean; + replacementWorkspaceCreation?: string; }): Promise { // Step 1: Resolve environment const selectedEnvironment = await resolveEnvironment({ @@ -370,6 +371,7 @@ export function environmentRunOrchestrator( executionWorkspaceSettings: input.executionWorkspaceSettings, runnerTaskProjectIds: input.runnerTaskProjectIds, runnerTaskResumeSessionId: input.runnerTaskResumeSessionId, + replacementWorkspaceCreation: input.replacementWorkspaceCreation, adapterType: input.adapterType ?? null, warmRunnerIdleTimeoutMs, runnerReconnectGraceMs, diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index b637afc081f..3c66e95c0a5 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -484,6 +484,7 @@ export interface EnvironmentDriverAcquireInput { executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; runnerTaskProjectIds?: readonly string[]; runnerTaskResumeSessionId?: string | null; + replacementWorkspaceCreation?: string; /** * The harness/adapter type for this run (the agent's adapter). Drivers that * materialize a per-run sandbox use it to select the runtime image so a single @@ -3473,27 +3474,43 @@ function createPluginEnvironmentDriver( }; } - async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) { - await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId))); + async function cleanupPluginRunLease(input: { environment: Environment | null; lease: EnvironmentLease }, + executionStopObserved = false) { const settled = await settleDispatchedEnvironmentTask(db, workerManager, input.lease, { executionStopObserved }); + const [current] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id), + eq(environmentLeases.companyId, input.lease.companyId))); + if (!current || current.providerLeaseId !== input.lease.providerLeaseId) throw new Error("Plugin lease cleanup ownership changed"); + const lease = toEnvironmentLeaseSnapshot(current); + const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease({ ...input, lease }); + const environmentId = input.environment?.id ?? readString(lease.metadata?.environmentId); + if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity"); + const receipt = await workerManager.call(plugin.id, lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", { + driverKey, companyId: lease.companyId, environmentId, issueId: lease.issueId, + config: driverConfig, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata ?? undefined, + }); const taskStopped = settled !== false || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); - if (settled !== "completed") { - const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); - await workerManager.call(plugin.id, "environmentReleaseLease", { - driverKey, - companyId: input.lease.companyId, - environmentId: input.environment.id, - issueId: input.lease.issueId, - config: driverConfig, - providerLeaseId: input.lease.providerLeaseId, - leaseMetadata: input.lease.metadata ?? undefined, - }); + (lease.metadata?.runnerTaskLease === true && lease.metadata?.runnerTaskDispatched !== true && !lease.metadata?.runnerTaskStartup && !lease.metadata?.runnerTaskStartupCleanup); + return remoteTerminationReceipt(lease, receipt) ? receipt + : taskStopped ? { providerLeaseId: lease.providerLeaseId, state: "stopped" } : undefined; + } + + async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) { + const scope = and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId)); + await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(scope); + let receipt: Awaited>; + try { + receipt = await cleanupPluginRunLease(input, executionStopObserved); + } catch (error) { + await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", + failureReason: "release_cleanup_failed", releasedAt: new Date(), updatedAt: new Date(), + }).where(and(scope, inArray(environmentLeases.status, ["active", "retained"]), + eq(environmentLeases.providerLeaseId, input.lease.providerLeaseId!))), + { companyId: input.lease.companyId, runId: input.lease.heartbeatRunId ?? undefined }); + throw error; } return await environmentsSvc.releaseLease(input.lease.id, input.status, { cleanupStatus: "success", - ...(taskStopped ? { remoteExecutionTermination: remoteTerminationReceipt(input.lease, - { providerLeaseId: input.lease.providerLeaseId, state: "stopped" }) } : {}), + remoteExecutionTermination: remoteTerminationReceipt(input.lease, receipt), }); } @@ -3574,7 +3591,9 @@ function createPluginEnvironmentDriver( companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId, issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, provider: `plugin:${parsed.config.pluginKey}:${parsed.config.driverKey}`, pluginId: plugin.id, driverKey: parsed.config.driverKey, - config: {}, timeoutMs: rpcTimeoutMs, requestKey: fingerprint, + config: {}, timeoutMs: rpcTimeoutMs, + requestKey: input.replacementWorkspaceCreation && input.replacementWorkspaceCreation === input.executionWorkspaceId + ? `${fingerprint}:${input.executionWorkspaceId}` : fingerprint, metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: parsed.config.pluginKey, driverKey: parsed.config.driverKey, ...(input.purpose ? { purpose: input.purpose, loginCommandKey: input.loginCommandKey } : {}) }, }, identity => workerManager.call(plugin.id, "environmentAcquireLease", { @@ -3694,18 +3713,7 @@ function createPluginEnvironmentDriver( } catch { return false; } }, - async retryPendingSandboxTeardown(input) { - const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); - const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); - const environmentId = input.environment?.id ?? readString(input.lease.metadata?.environmentId); - if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity"); - const receipt = await workerManager.call(plugin.id, input.lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", { - driverKey, companyId: input.lease.companyId, environmentId, issueId: input.lease.issueId, - config: driverConfig, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, - }); - return taskStopped ? { providerLeaseId: input.lease.providerLeaseId, state: "stopped" } : receipt; - }, + retryPendingSandboxTeardown: cleanupPluginRunLease, async realizeWorkspace(input) { const { plugin, pluginKey, driverKey, driverConfig } = await resolvePluginDriverForRelease({ @@ -4045,6 +4053,7 @@ export function environmentRuntimeService( executionWorkspaceSettings?: IssueExecutionWorkspaceSettings | null; runnerTaskProjectIds?: readonly string[]; runnerTaskResumeSessionId?: string | null; + replacementWorkspaceCreation?: string; /** The agent's adapter type for this run (mixed-harness environments). */ adapterType?: string | null; resolveSnapshotIdentity?: boolean; @@ -4091,6 +4100,7 @@ export function environmentRuntimeService( executionWorkspaceSettings: input.executionWorkspaceSettings ?? null, runnerTaskProjectIds: input.runnerTaskProjectIds, runnerTaskResumeSessionId: input.runnerTaskResumeSessionId, + replacementWorkspaceCreation: input.replacementWorkspaceCreation, purpose: input.purpose, loginCommandKey: input.loginCommandKey, adapterType: input.adapterType ?? null, diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index 87020bb274f..f25be7aa4e9 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -565,13 +565,15 @@ export async function executeEnvironmentTask(db: Db, workers: Pick tx.update(environmentLeases).set({ - ...(owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}), + ...(owner.id !== lease.id && owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}), metadata:sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({remoteExecutionTermination:receipt,runnerTaskExecutionStopped:true})}::jsonb`, }).where(and(eq(environmentLeases.id,owner.id),eq(environmentLeases.providerLeaseId,taskId))), { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined }); } } await db.update(environmentLeases).set({ - metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskDispatched":false}'::jsonb`, + metadata: sql`(case when ${environmentLeases.metadata}->>'runnerTaskPreparationStopped' = 'true' + then coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'runnerTaskStartup' + else coalesce(${environmentLeases.metadata}, '{}'::jsonb) end) || '{"runnerTaskDispatched":false}'::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.providerLeaseId, taskId), sql`${environmentLeases.metadata}->>'runnerTaskExecutionStopped' = 'true'`)); } @@ -597,8 +599,11 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskRetiring":true}'::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId))) .returning({ metadata: environmentLeases.metadata, heartbeatRunId: environmentLeases.heartbeatRunId, taskId: environmentLeases.providerLeaseId }); - const startup = current?.metadata?.runnerTaskStartup; const startupCleanup = current?.metadata?.runnerTaskStartupCleanup as { requiresPreparationStop?: boolean } | undefined; + const startup = current?.metadata?.runnerTaskStartup; + if (startupCleanup && current?.metadata?.runnerTaskExecutionStopped === true && current.metadata.runnerTaskDispatched === false && + !current.metadata.runnerTaskAdmissionRejection && !startup && + (!startupCleanup.requiresPreparationStop || current.metadata.runnerTaskPreparationStopped === true)) return "completed"; if (current?.metadata?.runnerTaskDispatched !== true && !startup && !startupCleanup) { if (current?.metadata) await settlePluginTaskCredentialHandoff(db, { companyId: lease.companyId, leaseId: lease.id, expectedTaskId: current.taskId ?? undefined, diff --git a/server/src/services/fresh-workspace-continuation.ts b/server/src/services/fresh-workspace-continuation.ts index 710147a2bfd..5baa71ba0d2 100644 --- a/server/src/services/fresh-workspace-continuation.ts +++ b/server/src/services/fresh-workspace-continuation.ts @@ -20,6 +20,7 @@ import { taskRetryRunAncestry } from "./recovery-run-ancestry.js"; import { parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js"; import { nativeExecutionUsesTaskWorkspace } from "./heartbeat/external-provider.js"; import { preparePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; +import { findWorkspaceProviderFailure } from "./workspace-provider-failure.js"; import { readFreshContinuationReadiness } from "./fresh-continuation-fallback.js"; export type ContinuationReadiness = @@ -522,6 +523,9 @@ export async function verifyRetryWorkspace( issueId: string, responsibleUserId: string | null, readiness: ContinuationReadiness, + options?: { + selectCandidate?: (workspace: typeof executionWorkspaces.$inferSelect) => Promise; + }, ): Promise { if (readiness.mode !== "fresh_isolated_workspace" || run.contextSnapshot?.taskRetryRequestId !== readiness.retryIntentId) return readiness; @@ -541,7 +545,7 @@ export async function verifyRetryWorkspace( companyId: run.companyId, issueId, agentId: run.agentId, successorRunId: run.id, source, cause: null, evidence: null, sourceRetryIntentId: source.contextSnapshot?.taskRetryRequestId, - verifyProviderWorkspace: workers ? async workspace => { + verifyProviderWorkspace: options?.selectCandidate ?? (workers ? async workspace => { try { return Boolean(await preparePluginWorkspace(db, workers, { companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id, responsibleUserId, @@ -549,15 +553,22 @@ export async function verifyRetryWorkspace( existingWorkspaceId: workspace.id, mode: workspace.mode, strategy: workspace.strategyType, verifyExisting: true, })); - } catch { + } catch (error) { + const failure = findWorkspaceProviderFailure(error); + if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error; return false; } - } : undefined, + } : undefined), }); if (!retained) return readiness; const reuse: ContinuationReadiness = { mode: "reuse_issue_workspace", retryIntentId: readiness.retryIntentId, sourceRunId: source.id, workspaceId: retained.id, cwd: retained.cwd }; - await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery", - action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: reuse }); + if (!options?.selectCandidate) await recordVerifiedRetryWorkspace(db, run, issueId, reuse); return reuse; } + +export async function recordVerifiedRetryWorkspace(db: Db, run: { companyId: string; id: string }, issueId: string, + readiness: Extract) { + await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery", + action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: readiness }); +} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 12de5b29fab..7866f601d3a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -34,7 +34,7 @@ import { registerCancelledWorkspacePreservationWork } from "./native-runtime/nat import { readUpgradeReadiness } from "./heartbeat/upgrade-readiness.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; -import { readFreshWorkspaceReadiness, verifyRetryWorkspace } from "./fresh-workspace-continuation.js"; +import { readFreshWorkspaceReadiness, verifyRetryWorkspace, type ContinuationReadiness } from "./fresh-workspace-continuation.js"; import { settleFailedNativeRecoveryPreparation } from "./native-runtime/native-recovery-preparation-failure.js"; import { promptInjectionPreventionService } from "./prompt-injection-prevention.js"; import { createHeartbeatAgentStatus } from "./heartbeat/agent-status.js"; @@ -49,7 +49,7 @@ import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkin import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral } from "./heartbeat/agent-directory-busy.js"; import { preparePredecessorRetirementFallback } from "./heartbeat/predecessor-retirement-fallback.js"; import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js"; -import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; +import { bindHeartbeatRunnerHomeMode, buildHeartbeatWorkspaceConfiguration, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; import { adoptsLiveRemoteRunner } from "./heartbeat/live-runner-adoption.js"; import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js"; import { watchShutdownHandoff } from "./heartbeat/shutdown-handoff.js"; @@ -60,7 +60,7 @@ import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heart import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js"; -import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; +import { prepareHeartbeatWorkspace, RetryWorkspaceRefused } from "./heartbeat/workspace-preparation.js"; import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js"; import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; @@ -2565,10 +2565,20 @@ export function heartbeatService( // Initialization has persisted the active context, including an explicit // absence of identity inherited from an automatic continuation. responsibleUserId = identityContext.responsibleUserId; - if (issueId && freshWorkspace?.mode === "fresh_isolated_workspace") { - const fallback = freshWorkspace; - freshWorkspace = await timePreparation("workspace_verification", () => - verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, fallback)); + const freshRetryFallback = freshWorkspace?.mode === "fresh_isolated_workspace" ? freshWorkspace : undefined; + let retryWorkspaceCandidate: Extract | undefined; + if (issueId && freshRetryFallback && !externalProvider && + parseObject(context.retryWorkspaceRefusal).intentId !== freshRetryFallback.retryIntentId) { + let verifyManagedWorkspace = false; + const retained = await timePreparation("workspace_verification", () => + verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, freshRetryFallback, + agent.adapterType === "paperclip_runner" && options.pluginWorkerManager + ? { selectCandidate: async () => { verifyManagedWorkspace = true; return true; } } : undefined)); + if (retained.mode === "reuse_issue_workspace") { + freshWorkspace = retained; + if (verifyManagedWorkspace) + retryWorkspaceCandidate = retained; + } } run = { ...run, @@ -2753,29 +2763,33 @@ export function heartbeatService( legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, }); - const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace" - ? "isolated_workspace" : trustPreset.kind === "low_trust_review" && - resolvedExecutionWorkspaceMode === "shared_workspace" - ? "isolated_workspace" - : resolvedExecutionWorkspaceMode; - const issueRef = issueContext - ? { - id: issueContext.id, - identifier: issueContext.identifier, - title: issueContext.title, - status: issueContext.status, - priority: issueContext.priority, - workMode: issueContext.workMode, - conversationAgentId: issueContext.conversationAgentId, - conversationUserId: issueContext.conversationUserId, - reviewPolicy: issueContext.reviewPolicy, - description: issueContext.description, - projectId: issueContext.projectId, - projectWorkspaceId: issueContext.projectWorkspaceId, - executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId, - executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference, - } - : null; + const workspaceInputs = () => { + const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace" + ? "isolated_workspace" : trustPreset.kind === "low_trust_review" && + resolvedExecutionWorkspaceMode === "shared_workspace" + ? "isolated_workspace" + : resolvedExecutionWorkspaceMode; + const issueRef = issueContext + ? { + id: issueContext.id, + identifier: issueContext.identifier, + title: issueContext.title, + status: issueContext.status, + priority: issueContext.priority, + workMode: issueContext.workMode, + conversationAgentId: issueContext.conversationAgentId, + conversationUserId: issueContext.conversationUserId, + reviewPolicy: issueContext.reviewPolicy, + description: issueContext.description, + projectId: issueContext.projectId, + projectWorkspaceId: issueContext.projectWorkspaceId, + executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId, + executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference, + } + : null; + return { requestedExecutionWorkspaceMode, issueRef }; + }; + let { requestedExecutionWorkspaceMode, issueRef } = workspaceInputs(); const storedLedgerScope = parseObject(parseObject(run.usageJson).ledgerScope); const runLedgerScope = Object.keys(storedLedgerScope).length > 0 ? storedLedgerScope @@ -2791,18 +2805,144 @@ export function heartbeatService( policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) }, services: { issuesSvc, getPinnedSkillTestContext }, })); - const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({ - agentConfig: config, - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - mode: requestedExecutionWorkspaceMode, - legacyUseProjectWorkspace: - issueAssigneeOverrides?.useProjectWorkspace ?? null, - adapterConfigOverrides: { - ...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))), - ...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}), + const selectWorkspace = async () => { + const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({ + agentConfig: config, + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, + mode: requestedExecutionWorkspaceMode, + legacyUseProjectWorkspace: + issueAssigneeOverrides?.useProjectWorkspace ?? null, + adapterConfigOverrides: { + ...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))), + ...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}), + }, + }); + const selection = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, { + run, agent, + task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace, + freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId }, + policy: { + trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, + projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, + }, + config: { config, sourceWorkspaceConfig, issueAssigneeOverrides }, + services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder }, + })); + return { ...selection, sourceWorkspaceConfig }; + }; + let workspaceSelection = await selectWorkspace(); + const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { + run, agent, + task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest, + adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) }, + config: { sourceWorkspaceConfig: workspaceSelection.sourceWorkspaceConfig, aiBinding }, + policy: { trustPreset, useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory, + explicitTaskDirectory: workspaceSelection.explicitTaskDirectory, requestedExecutionWorkspaceMode }, + environment: workspaceSelection, + services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral }, + onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; }, + onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; }, + onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; }, + })); + if (!executionConfiguration.prepared) return; + let { mergedConfig, configSnapshot } = executionConfiguration; + const { runScopedMentionedSkillKeys, githubSelection, useHostGitHub, + resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, + runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability, + } = executionConfiguration; + let { runtimeConfig } = executionConfiguration; + const resolvedFailureSecrets = readFailureReportSecrets(); + readFailureReportSecrets = () => [ + ...resolvedFailureSecrets, + ...identityRedactor.values, + ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys), + ]; + const prepareSession = () => timePreparation("session", () => prepareHeartbeatSession(db, { + run, agent, + task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision }, + session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec }, + config: { + runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides, + managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext, }, - }); + workspace: { + ...workspaceSelection, requestedExecutionWorkspaceMode, projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, + }, + environment: workspaceSelection, + getLatestAgentConfigRevision, + })); + let preparedSession = await prepareSession(); + const prepareWorkspace = () => timePreparation("workspace", () => prepareHeartbeatWorkspace(db, { + run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal, + task: { + issueRef, + issueId, + wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, + context, + responsibleUserId, + previousSessionParams: preparedSession.previousSessionParams, + retryWorkspaceCandidate, + workspaceSelection: taskWorkspaceIntentRow?.intent ?? null, + freshWorkspace, + }, + policy: { + trustPreset, + isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, + effectiveExecutionWorkspaceMode: workspaceSelection.effectiveExecutionWorkspaceMode, + requestedExecutionWorkspaceMode, + useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory, + nativeChatWorkspaceScope: workspaceSelection.nativeChatWorkspaceScope, + projectExecutionWorkspacePolicy, + issueExecutionWorkspaceSettings, + environmentExecutionWorkspaceSettings, + executionPolicy: workspaceSelection.executionPolicy, + }, + environment: workspaceSelection, + config: { + mergedConfig, + configSnapshot, + secretManifest, + managedAiCredentialIdentity: managedAiRuntime?.identity, + runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId, + }, + reuse: { + ...workspaceSelection, + runOptions, + }, + services: { + workspaceWorkers: options.pluginWorkerManager, + envOrchestrator, + executionWorkspacesSvc, + workspaceOperationsSvc, + resolveWorkspaceForRun, + resolveReusedGitWorkspaceAnchor, + appendRunEvent, + }, + })); + let preparedWorkspace; + try { + preparedWorkspace = await prepareWorkspace(); + } catch (error) { + if (!(error instanceof RetryWorkspaceRefused) || !freshRetryFallback) throw error; + retryWorkspaceCandidate = undefined; + freshWorkspace = freshRetryFallback; + ({ requestedExecutionWorkspaceMode, issueRef } = workspaceInputs()); + const selected = await selectWorkspace(); + if (selected.selectedEnvironmentId !== workspaceSelection.selectedEnvironmentId) + throw new Error("Retry environment changed during workspace verification"); + workspaceSelection = selected; + ({ mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration( + { ...selected.sourceWorkspaceConfig, homeMode: mergedConfig.homeMode }, + { trustPreset, useIsolatedTaskDirectory: selected.useIsolatedTaskDirectory, + explicitTaskDirectory: selected.explicitTaskDirectory, requestedExecutionWorkspaceMode }, + selected.selectedEnvironmentId)); + runtimeConfig = { ...runtimeConfig }; + if (Object.hasOwn(mergedConfig, "workspaceStrategy")) runtimeConfig.workspaceStrategy = mergedConfig.workspaceStrategy; + else delete runtimeConfig.workspaceStrategy; + preparedSession = await prepareSession(); + preparedWorkspace = await prepareWorkspace(); + } const { persistedRunnerProfile, persistedNativeExecutionInput, @@ -2825,58 +2965,9 @@ export function heartbeatService( materializeNativeChatWorktree, explicitTaskDirectory, useIsolatedTaskDirectory, - } = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, { - run, agent, - task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace, - freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId }, - policy: { - trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, - projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, - }, - config: { config, sourceWorkspaceConfig, issueAssigneeOverrides }, - services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder }, - })); + sourceWorkspaceConfig, + } = workspaceSelection; let nativeChatExpectedCwd = selectedNativeChatExpectedCwd; - const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { - run, agent, - task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest, - adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) }, - config: { sourceWorkspaceConfig, aiBinding }, - policy: { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }, - environment: { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput }, - services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral }, - onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; }, - onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; }, - onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; }, - })); - if (!executionConfiguration.prepared) return; - const { - mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub, - resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, - runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability, - } = executionConfiguration; - let { runtimeConfig } = executionConfiguration; - const resolvedFailureSecrets = readFailureReportSecrets(); - readFailureReportSecrets = () => [ - ...resolvedFailureSecrets, - ...identityRedactor.values, - ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys), - ]; - const preparedSession = await timePreparation("session", () => prepareHeartbeatSession(db, { - run, agent, - task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision }, - session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec }, - config: { - runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides, - managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext, - }, - workspace: { - requestedExecutionWorkspaceMode, effectiveExecutionWorkspaceMode, projectExecutionWorkspacePolicy, - issueExecutionWorkspaceSettings, executionPolicy, requestedReusableExecutionWorkspaceConfig, reusableExistingExecutionWorkspace, - }, - environment: { environmentResolution, selectedEnvironmentId, selectedEnvironmentForConfig }, - getLatestAgentConfigRevision, - })); const { sessionConfigMetadata, configuredModel, resetTaskSession, sessionResetReason, taskSessionForRun, getFreshSessionHandoff, previousSessionParams } = preparedSession; const { resolvedWorkspace, @@ -2898,64 +2989,7 @@ export function heartbeatService( workspaceRealization, executionTarget, workspaceRestoreSource, - } = await timePreparation("workspace", () => prepareHeartbeatWorkspace(db, { - run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal, - task: { - issueRef, - issueId, - wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, - context, - responsibleUserId, - previousSessionParams, - workspaceSelection: taskWorkspaceIntentRow?.intent ?? null, - freshWorkspace, - }, - policy: { - trustPreset, - isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, - effectiveExecutionWorkspaceMode, - requestedExecutionWorkspaceMode, - useIsolatedTaskDirectory, - nativeChatWorkspaceScope, - projectExecutionWorkspacePolicy, - issueExecutionWorkspaceSettings, - environmentExecutionWorkspaceSettings, - executionPolicy, - }, - environment: { - selectedEnvironmentId, - localEnvironment, - selectedEnvironmentForConfig, - environmentResolution, - resolvedInstanceSettings, - }, - config: { - mergedConfig, - configSnapshot, - secretManifest, - managedAiCredentialIdentity: managedAiRuntime?.identity, - runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId, - }, - reuse: { - requestedShouldReuseExisting, - existingExecutionWorkspace, - reusableExistingExecutionWorkspace, - workspaceReuseRequest, - nativeRecoveryExecutionWorkspaceId, - persistedNativeExecutionInput, - isDotRun, - runOptions, - }, - services: { - workspaceWorkers: options.pluginWorkerManager, - envOrchestrator, - executionWorkspacesSvc, - workspaceOperationsSvc, - resolveWorkspaceForRun, - resolveReusedGitWorkspaceAnchor, - appendRunEvent, - }, - })); + } = preparedWorkspace; const preparedInstructions = instructionWorkingCopy = createHeartbeatInstructionWorkingCopy({ observabilityCompatibilityKey: observability?.compatibilityKey, db, instructionCopies, readRun: () => run!, agent, context, diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 56fc9ca1a06..248cd400687 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -94,6 +94,23 @@ export interface HeartbeatExecutionConfigurationInput { onIdentityRedactorPrepared: (redactor: ReturnType) => void; } +export function buildHeartbeatWorkspaceConfiguration( + sourceWorkspaceConfig: Record, + { useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }: HeartbeatExecutionConfigurationInput["policy"], + selectedEnvironmentId: string | null, +) { + const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && ( + requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0 + )); + const mergedConfig: Record = { + ...sourceWorkspaceConfig, + // The base below is already task-owned. Keep directory transport while + // preserving isolated mode and the mandatory sandbox preflight. + ...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}), + }; + return { mergedConfig, configSnapshot: buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId) }; +} + /** Assembles effective run configuration; the executor retains cleanup ownership. */ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: HeartbeatExecutionConfigurationInput) { const { run, agent, onFailureSecretsResolved, onManagedAiRuntimeAcquired, onIdentityRedactorPrepared } = input; @@ -137,22 +154,10 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear // preparation step can fail. Subsequent provider/identity values extend it. onFailureSecretsResolved(() => []); const aiBinding = preconfiguredHome ? undefined : input.config.aiBinding; - const { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode } = input.policy; + const { trustPreset } = input.policy; const { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral } = input.services; let managedAiRuntime: ManagedAiRuntime | undefined; - const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && ( - requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0 - )); - const mergedConfig = { - ...sourceWorkspaceConfig, - // The base below is already task-owned. Keep directory transport while - // preserving isolated mode and the mandatory sandbox preflight. - ...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}), - }; - const configSnapshot = buildExecutionWorkspaceConfigSnapshot( - mergedConfig, - selectedEnvironmentId, - ); + const { mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration(sourceWorkspaceConfig, input.policy, selectedEnvironmentId); const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index bab1110db5a..c9d5bd9224e 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1797,7 +1797,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // path uses the provider and configuration recorded on the lease first; // the environment is lifecycle context and only a legacy fallback. const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral"; - const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease); + const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease) || + (metadata.driver === "plugin" && metadata.pluginLeaseCleanupMethod === "release"); // Do not consume a finite cleanup attempt while the provider plugin is // briefly unavailable. A plugin worker restart, a plugin reload, or a diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 06b3ae06a1e..d2039d644f3 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -1,3 +1,4 @@ +import { preparePluginWorkspace } from "../plugin-workspaces.js"; import { execFile as execFileCallback } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdtemp, mkdir, readFile, readdir, rm, writeFile, access, rename, symlink } from "node:fs/promises"; @@ -23,7 +24,7 @@ import { instanceSettingsService } from "../instance-settings.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { nativeWorkspaceSyncInternals, prepareNativeWorkspaceSync } from "../native-runtime/native-workspace-sync.js"; import { createHeartbeatWorkspaceResolver, prepareProjectRepositoryWorkspaces, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; -import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js"; +import { RetryWorkspaceRefused, prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js"; import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; import * as managedRepositoryCheckout from "../managed-repository-checkout.js"; @@ -965,15 +966,18 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replace: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replaceAfterPrepare: true }, - ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare }) => { + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, refuse: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, restart: true }, + ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare, verify: $verify, refuse: $refuse, restart: $restart)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare, verify, refuse, restart }) => { const { input, source, issue } = await fixture(true); if (portable) input.agent.adapterType = "paperclip_runner"; - const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : ""}`; + const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : refuse ? "-refuse" : restart ? "-restart" : verify ? "-verify" : ""}`; const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: "test-storage", version: "1.0.0", apiVersion: 1, status: "ready", categories: ["workspace"], manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Storage", description: "Storage", author: "Test", categories: ["workspace"], - workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], + workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, workspaceVerificationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" }, - ...(portable ? { sessionHandoffVersion: 1 as const } : {}) }], + ...(portable ? { sessionHandoffVersion: 1 as const, freshWorkspaceVersion: 1 as const } : {}) }], resourceLifecycle: ["project"], entrypoints: { worker: "./worker.js" } } }).returning(); const [environment] = await db.insert(environments).values({ name: `Storage tasks ${pluginKey}`, driver: "plugin", status: "active", config: { pluginKey, driverKey: "tasks", driverConfig: {} } }).returning(); @@ -1001,15 +1005,20 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace const prepareRequests: Array<{ operation: Parameters[2]["operation"]; operationId: string }> = []; const finalizationIds: string[] = []; let executionWorkspaceId: string | undefined; - input.services.workspaceWorkers = { getWorker: () => ({ supportedMethods: [ - "workspaceOperation", "environmentTask", ...(portable ? ["workspacePrepare"] : []), + let loseVerificationReply = restart === true; + input.services.workspaceWorkers = { isRunning: () => true, getWorker: () => ({ supportedMethods: [ + "workspaceOperation", "environmentTask", "environmentReleaseLease", ...(portable ? ["workspacePrepare"] : []), ] }), call: async (_: string, method: string, params: any) => { if (method === "environmentAcquireLease") { - expect(replace).toBe(true); + expect(replace || restart).toBe(true); expect(params.executionWorkspaceId).toBe(executionWorkspaceId); - sequence.push("replace"); + sequence.push(restart ? "allocate" : "replace"); return { providerLeaseId: readyTaskId }; } + if (method === "environmentReleaseLease") { + sequence.push("release"); + return { providerLeaseId: params.providerLeaseId, state: "stopped" }; + } if (method === "environmentTask") { if (replace && params.operation.kind !== "describe") { expect(params.taskId).toBe(providerLeaseId); @@ -1023,28 +1032,49 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace sequence.push("placement"); return { kind: "description", taskId: readyTaskId, runner: { version: "test", digest: `sha256:${"a".repeat(64)}` } }; } - expect(params.taskLease).toEqual(portable ? taskLease : undefined); + expect(params.taskLease).toEqual(portable && (!verify || executionWorkspaceId) ? taskLease : undefined); if (params.operation.kind === "finalize") { sequence.push("finalize"); finalizationIds.push(params.operationId); return { kind: "finalized", headSha: "a".repeat(40), branchName: "test-WSP-1" }; } - if (portable) { + if (params.operation.verifyExisting) { + expect(method).toBe("workspaceOperation"); + sequence.push("workspace verify"); + if (loseVerificationReply) { + loseVerificationReply = false; + throw new Error("workspace verification reply lost"); + } + } else if (portable && (!verify || executionWorkspaceId)) { expect(method).toBe("workspacePrepare"); expect(params.phase).toBe("submit"); expect(params.executionWorkspaceId).toBe(executionWorkspaceId); prepareRequests.push(params); } - sequence.push("workspace prepare"); + if (!params.operation.verifyExisting) sequence.push("workspace prepare"); if (replaceAfterPrepare && prepareRequests.length === 2) throw new Error("lost workspace preparation reply"); return { kind: "prepared", binding: { pluginId: plugin.id, projectId: issue.projectId, executionWorkspaceId: params.executionWorkspaceId, projectWorkspaceId: issue.projectWorkspaceId, sourceRevision: params.operation.source.revision, workspaceRef: replaceAfterPrepare && prepareRequests.length === 3 ? "provider:replacement-checkout" : "provider:checkout" }, - branchName: params.operation.branchName, baseSha: "a".repeat(40), created: true, branchCreatedByRuntime: true }; + branchName: params.operation.branchName, baseSha: "a".repeat(40), created: refuse || !params.operation.verifyExisting, branchCreatedByRuntime: true }; } } as any; input.services.envOrchestrator = environmentRunOrchestrator(db, { pluginWorkerManager: input.services.workspaceWorkers as never, }); + if (verify) { + const prepared = await preparePluginWorkspace(db, input.services.workspaceWorkers, { + companyId: input.agent.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, + issueId: issue.id, runId: input.run.id, agentId: input.agent.id, responsibleUserId: null, + existingWorkspaceId: null, mode: "isolated_workspace", strategy: "git_worktree", branchName: "test-WSP-1", + }); + const workspace = (await input.services.executionWorkspacesSvc.getStoredById(input.agent.companyId, prepared!.id))!; + input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id }; + input.reuse.existingExecutionWorkspace = input.reuse.reusableExistingExecutionWorkspace = workspace; + input.task.freshWorkspace = input.task.retryWorkspaceCandidate = { mode: "reuse_issue_workspace", + retryIntentId: randomUUID(), sourceRunId: randomUUID(), workspaceId: workspace.id, cwd: workspace.cwd }; + sequence.length = 0; + prepareRequests.length = 0; + } const acquire = input.services.envOrchestrator.acquireForRun; vi.spyOn(input.services.envOrchestrator, "acquireForRun").mockImplementation(async args => { sequence.push("acquire"); @@ -1054,6 +1084,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace } expect(args.persistedExecutionWorkspace).toMatchObject({ mode: defaultMode, id: expect.any(String) }); executionWorkspaceId = args.persistedExecutionWorkspace!.id; + if (restart) return acquire(args); const [lease] = await db.insert(environmentLeases).values({ companyId: args.companyId, environmentId: environment.id, heartbeatRunId: args.heartbeatRunId, issueId: args.issueId, executionWorkspaceId: args.persistedExecutionWorkspace!.id, provider: `plugin:${pluginKey}:tasks`, providerLeaseId, @@ -1089,7 +1120,50 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace } throw new Error("reached execution handoff"); }); - const portableSequence = ["acquire", ...(replace ? ["replace"] : []), "placement", "workspace prepare"]; + if (restart) { + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ kind: "transport", retryable: true }); + expect(sequence).toEqual(["acquire", "allocate", "workspace verify"]); + const [lease] = await leases(input.run.id); + expect(lease).toMatchObject({ providerLeaseId, status: "active" }); + expect((await readRun(input.run.id)).contextSnapshot?.retryWorkspaceRefusal).toBeUndefined(); + input.reuse.runOptions.nativeRestartRecovery = { + kind: "bootstrap_incomplete", runId: input.run.id, leaseOwner: "restarted-controller", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + }; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff"); + expect(sequence).toEqual(["acquire", "allocate", "workspace verify", "acquire", "workspace verify", "placement", "workspace prepare"]); + expect((await leases(input.run.id)).map(item => item.id)).toEqual([lease.id]); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskDispatched: true } }) + .where(eq(environmentLeases.id, lease.id)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("Workspace verification lease already started execution"); + expect(sequence.slice(-2)).toEqual(["workspace prepare", "acquire"]); + return; + } + if (refuse) { + const candidate = input.task.retryWorkspaceCandidate!; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow(RetryWorkspaceRefused); + expect(sequence).toEqual(["acquire", "workspace verify", "release"]); + const persisted = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.run.id)).then(([row]) => row); + expect(persisted.contextSnapshot?.retryWorkspaceRefusal).toMatchObject({ + intentId: candidate.retryIntentId, workspaceId: candidate.workspaceId, leaseId: expect.any(String), + }); + expect((await leases(input.run.id))[0]).toMatchObject({ status: "released", cleanupStatus: "success" }); + const next = new Error("fresh acquisition reached"); + vi.mocked(input.services.envOrchestrator.acquireForRun).mockRejectedValue(next); + await expect(prepareHeartbeatWorkspace(db, { + ...input, run: persisted, + reuse: { ...input.reuse, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null }, + task: { ...input.task, context: persisted.contextSnapshot!, retryWorkspaceCandidate: undefined, + issueRef: { ...input.task.issueRef!, executionWorkspaceId: null }, + freshWorkspace: { mode: "fresh_isolated_workspace", retryIntentId: candidate.retryIntentId, + sourceRunId: candidate.sourceRunId, retainedSourceIds: [] } }, + })).rejects.toBe(next); + expect(input.services.envOrchestrator.acquireForRun).toHaveBeenLastCalledWith(expect.objectContaining({ + replacementWorkspaceCreation: expect.any(String), persistedExecutionWorkspace: expect.objectContaining({ id: expect.not.stringMatching(candidate.workspaceId) }), + })); + return; + } + const portableSequence = ["acquire", ...(verify ? ["workspace verify"] : []), ...(replace ? ["replace"] : []), "placement", "workspace prepare"]; if (replaceAfterPrepare) { const { executionTarget } = await prepareHeartbeatWorkspace(db, input); if (executionTarget?.kind !== "remote" || executionTarget.transport !== "sandbox" || !executionTarget.runnerTask) diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index d437a8cc273..99403f393b7 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -5,7 +5,8 @@ import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, Remote import { WorkspaceBusyDeferral } from "./retries.js"; import { remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold.js"; import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js"; -import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js"; +import { findWorkspaceProviderFailure } from "../workspace-provider-failure.js"; +import { readFreshWorkspaceReadiness, recordVerifiedRetryWorkspace, type ContinuationReadiness } from "../fresh-workspace-continuation.js"; import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js"; import { NativeExecutionContext } from "../native-runtime/native-execution-context.js"; import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js"; @@ -39,9 +40,9 @@ import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import fs from "node:fs/promises"; import path from "node:path"; import { CONTROLLER_PROVISION_COPY_ONLY, workspaceRequiresCopyOnlyProvisioning } from "../workspace-host-provision.js"; -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, sql } from "drizzle-orm"; import { type ExecutionWorkspace } from "@paperclipai/shared"; -import { executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db"; +import { environmentLeases, executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db"; import { getStartupTracer } from "../../instrumentation.js"; import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js"; import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js"; @@ -103,6 +104,8 @@ type WorkspaceMetadataInput = Parameters | null; freshWorkspace?: import("../fresh-workspace-continuation.js").ContinuationReadiness | null; + retryWorkspaceCandidate?: Extract; workspaceSelection?: import("@paperclipai/shared").TaskWorkspaceIntent | null; }; policy: { @@ -161,7 +165,7 @@ export interface HeartbeatWorkspacePreparationInput { }; services: { workspaceWorkers?: WorkspaceWorkers; - envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun">; + envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun" | "releaseForRun">; executionWorkspacesSvc: Pick, "create" | "update" | "getStoredById" | "bindTaskWorkspace" | "prepareTaskRepositoriesForAdmission" | "assertTaskRepositoryPublicationAllowed" | "prepareConfiguredRepositoryCopy" | "assertTaskRepositoryRecoveryBinding">; workspaceOperationsSvc: Pick, "createRecorder">; resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"]; @@ -580,6 +584,29 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace throw new Error(allowlistDecision.reason); } }; + const refused = parseObject(context.retryWorkspaceRefusal); + const refusedWorkspaceId = fresh && input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && + refused.intentId === input.task.freshWorkspace.retryIntentId && typeof refused.workspaceId === "string" + ? refused.workspaceId : undefined; + const priorVerificationLease = refusedWorkspaceId && selectedEnvironmentForConfig && typeof refused.leaseId === "string" + ? (await db.select().from(environmentLeases).where(and( + eq(environmentLeases.id, refused.leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.environmentId, selectedEnvironmentForConfig.id), eq(environmentLeases.executionWorkspaceId, refusedWorkspaceId), + )))[0] : undefined; + if (refusedWorkspaceId && typeof refused.leaseId === "string" && !priorVerificationLease) + throw new Error("Workspace verification lease changed"); + const releaseVerificationLease = async (leaseId: string) => { + await controllerLease.assertOwned(); + await envOrchestrator.releaseForRun({ companyId: agent.companyId, agentId: agent.id, heartbeatRunId: run.id }); + const [lease] = await db.select({ status: environmentLeases.status, releasedAt: environmentLeases.releasedAt, + cleanupStatus: environmentLeases.cleanupStatus }).from(environmentLeases).where(and( + eq(environmentLeases.id, leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id))); + if (!lease?.releasedAt || !["released", "expired"].includes(lease.status) || lease.cleanupStatus !== "success") + throw new Error("Workspace verification lease cleanup is pending"); + }; + if (priorVerificationLease && (!priorVerificationLease.releasedAt || priorVerificationLease.cleanupStatus !== "success")) + await releaseVerificationLease(priorVerificationLease.id); + const retryCandidate = input.task.retryWorkspaceCandidate; const environmentAcquisitionInput = ( persistedExecutionWorkspace: Parameters[0]["persistedExecutionWorkspace"], projectId: string | null, @@ -596,46 +623,64 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace runnerTaskProjectIds: [...new Set([projectId, ...resolvedWorkspace.runnerTaskProjectIds ?? [], ...(additionalWorkspaces ?? []).map(workspace => workspace.projectId)] .filter((id): id is string => Boolean(id)))], - recoverPluginTask: !portablePreparationLeaseId && + recoverPluginTask: !priorVerificationLease && !portablePreparationLeaseId && + !(retryCandidate && restartRecovery?.kind === "bootstrap_incomplete") && (restartRecovery?.kind === "resume_dead_runner" || restartRecovery?.kind === "bootstrap_incomplete"), issueId: issueId ?? null, heartbeatRunId: run.id, agentId: agent.id, persistedExecutionWorkspace, + ...(priorVerificationLease && persistedExecutionWorkspace && persistedExecutionWorkspace.id !== refusedWorkspaceId + ? { replacementWorkspaceCreation: persistedExecutionWorkspace.id } : {}), executionWorkspaceSettings: environmentExecutionWorkspaceSettings, }); const pluginRecovery = restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.remote.kind === "plugin_task" - ? restartRecovery.remote : portablePreparationLeaseId + ? restartRecovery.remote : portablePreparationLeaseId && portablePreparationLeaseId !== priorVerificationLease?.id ? { kind: "plugin_task" as const, leaseId: portablePreparationLeaseId, preparation: true as const } : undefined; let workspaceAcquisition: Awaited> | undefined; let workspaceAcquireStartedAtMs: number | undefined; let ensureTaskWorkspace: ((taskId: string, assertOwned: () => void) => Promise | undefined) | undefined; - const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, { + const acquireTaskLease: NonNullable[2]["acquireTaskLease"]> = async workspace => { + if (workspaceAcquisition) { + const { environment, lease } = workspaceAcquisition; + if (lease.executionWorkspaceId !== workspace.id || typeof lease.metadata?.driverKey !== "string" || !lease.providerLeaseId) + throw new Error("Workspace placement changed"); + return { leaseId: lease.id, taskLease: { environmentId: environment.id, driverKey: lease.metadata.driverKey, providerLeaseId: lease.providerLeaseId } }; + } + if (!selectedEnvironmentForConfig) throw new Error("Workspace placement is unavailable"); + assertAllowedEnvironment(selectedEnvironmentForConfig); + await controllerLease.assertOwned(); + workspaceAcquireStartedAtMs = Date.now(); + if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id) + throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id"); + const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ + ...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces), + ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}), + })); + await controllerLease.assertOwned(); + if (retryCandidate) { + const metadata = acquisition.lease.metadata; + if (metadata?.runnerTaskDispatched === true || metadata?.runnerTaskIdentity || + metadata?.runnerTaskStartup || metadata?.runnerTaskStartupCleanup || metadata?.runnerTaskRetiring === true) + throw new Error("Workspace verification lease already started execution"); + workspaceAcquisition = acquisition; + return acquireTaskLease(workspace); + } + const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, { + companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal, + }); + await controllerLease.assertOwned(); + workspaceAcquisition = placement.acquisition; + return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease }; + }; + const pluginPrepareInput: Parameters[2] = { companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId, issueId, runId: run.id, agentId: agent.id, responsibleUserId, existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null, ...(providerFresh ? { fresh: providerFresh, expectedBindingRevision: previousProviderBindingRevision } : {}), selectedProviderPluginId: projectExecutionWorkspacePolicy?.storageProviderPluginId ?? null, environment: selectedEnvironmentForConfig, - acquireTaskLease: async workspace => { - if (!selectedEnvironmentForConfig) throw new Error("Workspace Launcher placement is unavailable"); - assertAllowedEnvironment(selectedEnvironmentForConfig); - await controllerLease.assertOwned(); - workspaceAcquireStartedAtMs = Date.now(); - if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id) - throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id"); - const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ - ...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces), - ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}), - })); - await controllerLease.assertOwned(); - const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, { - companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal, - }); - await controllerLease.assertOwned(); - workspaceAcquisition = placement.acquisition; - return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease }; - }, + acquireTaskLease, onTaskPrepared: ensure => { ensureTaskWorkspace = ensure; }, mode: requestedExecutionWorkspaceMode, strategy: resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig), @@ -647,7 +692,45 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace })) : null, baseRef: typeof parseObject(mergedConfig.workspaceStrategy).baseRef === "string" ? String(parseObject(mergedConfig.workspaceStrategy).baseRef) : null, - }); + }; + if (retryCandidate && !controllerOnly && !remoteAuthority) { + const verified = await timeWorkspace("workspace_verification", () => preparePluginWorkspace(db, input.services.workspaceWorkers, { + ...pluginPrepareInput, + existingWorkspaceId: retryCandidate.workspaceId, + verifyExisting: true, + assertOwned: () => input.signal?.throwIfAborted(), + onTaskPrepared: undefined, + })).catch(error => { + const failure = findWorkspaceProviderFailure(error); + if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error; + return null; + }); + await controllerLease.assertOwned(); + if (!verified) { + const refusal = { intentId: retryCandidate.retryIntentId, workspaceId: retryCandidate.workspaceId, + leaseId: workspaceAcquisition?.lease.id ?? null }; + const [saved] = await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) + || ${JSON.stringify({ retryWorkspaceRefusal: refusal })}::jsonb` }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.status, "running"))) + .returning({ id: heartbeatRuns.id }); + if (!saved) throw new Error("Workspace verification lost run ownership"); + context.retryWorkspaceRefusal = refusal; + if (workspaceAcquisition) await releaseVerificationLease(workspaceAcquisition.lease.id); + throw new RetryWorkspaceRefused(); + } + await recordVerifiedRetryWorkspace(db, run, issueId!, retryCandidate); + if (workspaceAcquisition && selectedEnvironmentForConfig) { + const providerLeaseId = workspaceAcquisition.lease.providerLeaseId; + const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(workspaceAcquisition, { + companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: retryCandidate.workspaceId, signal: input.signal, + }); + if (placement.acquisition.lease.providerLeaseId !== providerLeaseId) + throw new Error("Workspace placement changed after verification"); + workspaceAcquisition = placement.acquisition; + } + } + const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, pluginPrepareInput); + const freshTaskReservationId = providerFresh && pluginTaskWorkspace && !pluginPrepared && issueId ? await reserveFreshPluginTaskWorkspace({ db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id, fresh: providerFresh, responsibleUserId, expectedBindingRevision: previousProviderBindingRevision, owner: taskWorkspaceOwner!, diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index 67c9008b628..fa8efcbc9f9 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -86,6 +86,7 @@ export async function preparePluginWorkspace( fresh?: PluginFreshWorkspace; expectedBindingRevision?: number; verifyExisting?: true; + assertOwned?: () => void; acquireTaskLease?: (workspace: Pick) => Promise<{ leaseId: string; taskLease: TaskPlacement }>; onTaskPrepared?: (ensure: EnsureTaskWorkspace) => void; }, @@ -165,6 +166,7 @@ export async function preparePluginWorkspace( const selectedProvider = await requireWorkspaceProviderWorker(db, workers, input.companyId, pluginId, true); if (input.environment) assertProjectStorageEnvironment(selectedProvider, input.environment); const declaration = selectedProvider.manifestJson.workspaceProvider!; + if (input.verifyExisting && declaration.workspaceVerificationVersion !== 1) return null; if (input.fresh && declaration.freshWorkspaceVersion !== 1) throw conflict("A fresh attempt is unavailable for this workspace storage provider. The existing workspace is retained.", { reason: "fresh_workspace_storage_unsupported" }); if ( @@ -348,7 +350,7 @@ export async function preparePluginWorkspace( driver.driverKey === driverKey && driver.sessionHandoffVersion === 1) ? await input.acquireTaskLease?.({ id: workspace.id, mode: mode as ExecutionWorkspace["mode"] }) : undefined; const prepare = (workspace: Workspace, taskLease?: TaskPlacement, replacement?: { leaseId: string; assertOwned: () => void }) => withExecutionWorkspaceLifecycleLock(db, workspace.id, async (tx) => { - replacement?.assertOwned(); + (replacement?.assertOwned ?? input.assertOwned)?.(); const [current] = await tx .select() .from(executionWorkspaces) @@ -401,7 +403,7 @@ export async function preparePluginWorkspace( runId: input.runId, agentId: input.agentId, taskLease, - assertOwned: replacement?.assertOwned, + assertOwned: replacement?.assertOwned ?? input.assertOwned, operation: { kind: "prepare", source: sourceInput, @@ -430,8 +432,9 @@ export async function preparePluginWorkspace( (saved && result.binding.workspaceRef !== saved.workspaceRef) || (workspace.branchName !== null && result.branchName !== workspace.branchName) || (typeof savedBase === "string" && result.baseSha !== savedBase))) { - throw conflict("The existing checkout could not be verified"); + return null; } + (replacement?.assertOwned ?? input.assertOwned)?.(); await authorize(); const [currentProject] = await tx .select() @@ -485,12 +488,13 @@ export async function preparePluginWorkspace( return updated!; }); const prepared = await prepare(workspace, placement?.taskLease); - if (placement && input.onTaskPrepared) { + if (prepared && placement && input.onTaskPrepared) { let preparedTaskId = placement.taskLease.providerLeaseId; let preparedWorkspace = prepared; input.onTaskPrepared((taskId, assertOwned) => { if (taskId === preparedTaskId) return; return prepare(preparedWorkspace, { ...placement.taskLease, providerLeaseId: taskId }, { leaseId: placement.leaseId, assertOwned }).then(updated => { + if (!updated) throw conflict("Workspace preparation is incomplete"); preparedWorkspace = updated; preparedTaskId = taskId; });