diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index 4ca856513cd..3d52d0e11c8 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -204,12 +204,16 @@ A storage provider can declare `workspaceProvider.workspaceVerificationVersion: Before creating a fresh workspace for an admitted Retry, Core may send the original task-owned workspace's `prepare` request with `verifyExisting: true`. Core sends verification through `workspaceOperation` with a 10-second -deadline and a 12-second RPC timeout. +deadline and a 12-second RPC timeout. Drivers with task placement receive the +selected run's `taskLease` before Core describes the Runner task. Use that lease +to select the same execution placement for verification and later preparation. The provider must verify a completed checkout without creating or fetching one and return `prepared` with `created: false`. Core checks prior writer settlement and continues normal preparation on the retained workspace after -verification. A missing or unconfirmed checkout leaves the fresh retry -fallback in place. Local workspaces use the same ownership and writer +verification. Return the workspace failure reason `binding_mismatch` for a +definitively missing or invalid checkout. Core can then release the verification +lease and use the authorized fresh retry fallback. Transport failures and +timeouts keep the current checkout decision retryable. Local workspaces use the same ownership and writer checks with Core's existing local Git verification. A storage provider can declare `workspaceProvider.finalizationVersion: 1`. diff --git a/server/src/__tests__/plugin-runner-task.integration.test.ts b/server/src/__tests__/plugin-runner-task.integration.test.ts index fa7cebf6cdd..403eba17f2e 100644 --- a/server/src/__tests__/plugin-runner-task.integration.test.ts +++ b/server/src/__tests__/plugin-runner-task.integration.test.ts @@ -416,7 +416,7 @@ databaseSuite("Core plugin Runner task lifecycle", () => { return; } const next = await acquire; - expect(operations).toEqual(["stop", "status", "complete", "environmentAcquireLease"]); + expect(operations).toEqual(["stop", "status", "complete", "environmentReleaseLease", "environmentAcquireLease"]); expect(next.lease.id).not.toBe(oldLeaseId); await expect(resolvePluginTaskResume(db, { companyId, leaseId: next.lease.id, priorIdentity })).resolves.toBe("old-task"); await expect(runtime.task({ companyId, leaseId: next.lease.id, operation: { kind: "submit", projectIds: [], bootstrapTicket: "fixture", runner } })).rejects.toThrow("identity mismatch"); diff --git a/server/src/__tests__/plugin-warm-tasks.integration.test.ts b/server/src/__tests__/plugin-warm-tasks.integration.test.ts index 88d4d8605ff..e2da8aa0d80 100644 --- a/server/src/__tests__/plugin-warm-tasks.integration.test.ts +++ b/server/src/__tests__/plugin-warm-tasks.integration.test.ts @@ -23,6 +23,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport const companyId=randomUUID(),agentId=randomUUID(),issueId=randomUUID(),environmentId=randomUUID(),pluginId=randomUUID(); let runId=randomUUID(),allocation=0,stopped=false,outage=false,releaseFails=false,unavailable=false; const calls:string[]=[],dispatched:Array<[string,string]>=[],failures=new Set(); + const releaseMetadata: unknown[] = []; let projectIds=[...initialProjects],providerExpiry=initialExpiry; const taskExpiries=new Map(),acquireRequests:any[]=[]; let resumeGate:Promise|undefined,releaseResume:(()=>void)|undefined,resumes=0; @@ -47,7 +48,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport } if(method==="environmentResumeLease") {if(outage) throw Error("unreachable"); if(resumeGate){if(++resumes===2)releaseResume!();await resumeGate;}return {providerLeaseId:stopped||unavailable?null:p.providerLeaseId,expiresAt:taskExpiries.get(p.providerLeaseId),metadata:{runId:p.runId}};} - if(method==="environmentReleaseLease") {if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};} + if(method==="environmentReleaseLease") {releaseMetadata.push(p.leaseMetadata);if(releaseFails) throw Error("release unavailable");return {providerLeaseId:p.providerLeaseId,state:"stopped"};} if(p.operation.kind==="status") return {kind:"status",taskId:p.taskId,phase:stopped?"completed":"running",executionStopped:stopped,sessionStateAvailable:stopped,preparationStopped:stopped}; if(p.operation.kind==="read_state") return {kind:"state",taskId:p.taskId,state:{...p.operation.identity,lifecycle:"ready"}}; if(p.operation.kind==="stop") stopped=true; @@ -66,11 +67,11 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport launch:{runnerVersion:"fixture",runnerDigest:`sha256:${"a".repeat(64)}`,maxOutboxBytes:262144,p0ReserveBytes:65536,lifecyclePolicy:{mode:"warm",idleTimeoutMs:300000}}}}); await runtime().releaseRunLeases(runId,"released",undefined,"keep_running"); await db.update(heartbeatRuns).set({status:"succeeded"}).where(eq(heartbeatRuns.id,runId)); - return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests, + return {companyId,first,identity,calls,dispatched,runtime,acquire,acquireRequests,releaseMetadata, fail(kind:string){failures.add(kind);},unavailable(){unavailable=true;}, setExpiry(next?:string){providerExpiry=next;}, async nextRun(){runId=randomUUID();await db.insert(heartbeatRuns).values({id:runId,companyId,agentId,nativeIssueId:issueId,invocationSource:"manual",status:"running"});return runId;}, - setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},outage(){outage=true;}, + setProjects(next:string[]){projectIds=next;},stopped(){stopped=true;},failRelease(){releaseFails=true;},restoreRelease(){releaseFails=false;},outage(){outage=true;}, raceResume(){resumeGate=new Promise(resolve=>{releaseResume=resolve;});}}; } it("hands a retained process to a new run after controller reconstruction",async()=>{ @@ -118,7 +119,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport if(prepared) await db.update(environmentLeases).set({metadata:sql`${environmentLeases.metadata} || ${JSON.stringify({runnerTaskStartup:{contextId:"settled-startup",deadlineAtMs:1,runId:f.identity.runId}})}::jsonb`}).where(eq(environmentLeases.id,f.first.lease.id)); const calls=f.calls.length,dispatched=f.dispatched.length; const next=await f.acquire(); - expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentAcquireLease"]); + expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","complete","environmentReleaseLease","environmentAcquireLease"]); expect(f.dispatched.slice(dispatched)).toEqual([["status",f.first.lease.providerLeaseId],["complete",f.first.lease.providerLeaseId]]); expect(next.lease.providerLeaseId).not.toBe(f.first.lease.providerLeaseId); const [old]=await db.select().from(environmentLeases).where(eq(environmentLeases.id,f.first.lease.id)); @@ -129,7 +130,7 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport const f=await fixture();f.unavailable();await f.nextRun(); const calls=f.calls.length; await f.acquire(); - expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentAcquireLease"]); + expect(f.calls.slice(calls)).toEqual(["environmentResumeLease","status","stop","status","complete","environmentReleaseLease","environmentAcquireLease"]); }); it.each([["status",false],["complete",false],["stop",true]] as const)("keeps a stopped retained task for cleanup retry when %s fails",async(kind,running)=>{ const f=await fixture();if(running) f.unavailable(); else f.stopped();await f.nextRun(); @@ -143,6 +144,29 @@ const support=external ? {supported:true} : await getEmbeddedPostgresTestSupport await expect(f.runtime().retryPendingSandboxTeardown({environment:null,lease:pending})).resolves.toEqual({providerLeaseId:f.first.lease.providerLeaseId,state:"stopped"}); expect(f.calls.slice(calls)).toEqual(["stop","status","complete","environmentReleaseLease"]); }); + it("retries lease release after completed task settlement without repeating settlement", async () => { + const f = await fixture(); + f.stopped(); + f.failRelease(); + await f.nextRun(); + const before = f.calls.length; + await expect(f.acquire()).rejects.toThrow("release unavailable"); + expect(f.calls.slice(before)).toEqual(["environmentResumeLease", "status", "complete", "environmentReleaseLease"]); + const pending = (await environmentService(db).getLeaseById(f.first.lease.id))!; + expect(pending).toMatchObject({ + status: "pending_cleanup", cleanupStatus: "failed", + metadata: { runnerTaskDispatched: false, runnerTaskExecutionStopped: true }, + }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.identity.runId)); + expect(run.status).toBe("succeeded"); + f.restoreRelease(); + const retry = f.calls.length; + await expect(f.runtime().retryPendingSandboxTeardown({ environment: null, lease: pending })) + .resolves.toEqual({ providerLeaseId: f.first.lease.providerLeaseId, state: "stopped" }); + expect(f.calls.slice(retry)).toEqual(["environmentReleaseLease"]); + expect(f.releaseMetadata).toHaveLength(2); + for (const metadata of f.releaseMetadata) expect(metadata).toMatchObject({ runnerTaskDispatched: false, runnerTaskExecutionStopped: true }); + }); it("does not allocate a replacement when the retained computer cannot be inspected",async()=>{ const f=await fixture();f.outage();await f.nextRun(); await expect(f.acquire()).rejects.toThrow("unreachable"); diff --git a/server/src/__tests__/plugin-workspaces.integration.test.ts b/server/src/__tests__/plugin-workspaces.integration.test.ts index 7c21470b53d..a412bc0196f 100644 --- a/server/src/__tests__/plugin-workspaces.integration.test.ts +++ b/server/src/__tests__/plugin-workspaces.integration.test.ts @@ -1,3 +1,4 @@ +import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk"; import { isPluginWorkspaceReplay } from "../services/plugin-workspace-replay-context.js"; import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js"; import { randomUUID } from "node:crypto"; @@ -51,7 +52,7 @@ const support = await getEmbeddedPostgresTestSupport(); }); const fixture = (companyId?: string) => createPluginWorkspaceFixture(db, companyId); - it.each(["ready", "lost_reply", "unconfirmed", "live_writer", "unsupported"] as const)( + it.each(["ready", "lost_reply", "unconfirmed", "transport", "worker_lookup", "live_writer", "older_retained", "unsupported"] as const)( "verifies the same checkout before choosing a fresh retry (%s)", async (state) => { const f = await fixture(); const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId)); @@ -73,6 +74,15 @@ const support = await getEmbeddedPostgresTestSupport(); status: state === "live_writer" ? "active" : "released", releasedAt: state === "live_writer" ? null : new Date(), }); + if (state === "older_retained") { + const [older] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId, + issueId: f.issueId, status: "succeeded", finishedAt: new Date() }).returning(); + await db.insert(environmentLeases).values({ + companyId: f.companyId, issueId: f.issueId, heartbeatRunId: older.id, + provider: "plugin:test:tasks", executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_environment", + status: "retained", releasedAt: null, + }); + } const retryIntentId = randomUUID(); const [retry] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.input.agentId, issueId: f.issueId, status: "running", contextSnapshot: { taskRetryRequestId: retryIntentId }, @@ -80,24 +90,34 @@ const support = await getEmbeddedPostgresTestSupport(); const fresh = { mode: "fresh_isolated_workspace", retryIntentId, sourceRunId: f.runId, retainedSourceIds: [] } as const; const call = vi.fn(async (...args) => { const result = await f.workers.call(...args) as { kind: string }; - if (state === "unconfirmed") throw new Error("checkout incomplete"); + if (state === "transport") throw new Error("connection closed"); + if (state === "unconfirmed") throw new JsonRpcCallError({ + code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: "checkout incomplete", + data: workspaceOperationErrorData(new PluginWorkspaceOperationError("binding_mismatch")), + }); return { ...result, created: false }; }); const workers: WorkspaceWorkers = { getWorker: (pluginId) => { + if (state === "worker_lookup") throw new Error("worker lookup timed out"); const worker = f.workers.getWorker(pluginId); return worker && { ...worker, supportedMethods: ["workspaceOperation", "workspacePrepare"] }; }, call, }; const verificationStarted = Date.now(); - const result = await verifyRetryWorkspace(db, workers, retry, f.issueId, null, - { ...fresh, retainedSourceIds: [] }); + const result = verifyRetryWorkspace(db, workers, retry, f.issueId, null, fresh); + if (state === "transport" || state === "worker_lookup") { + if (state === "transport") await expect(result).rejects.toMatchObject({ kind: "transport", retryable: true }); + else await expect(result).rejects.toThrow("worker lookup timed out"); + return; + } const verified = state === "ready" || state === "lost_reply"; - expect(result).toEqual(verified + expect(await result).toEqual(verified ? { mode: "reuse_issue_workspace", retryIntentId, sourceRunId: f.runId, workspaceId, cwd: null } : fresh); - expect(f.calls.slice(1)).toHaveLength(state === "live_writer" || state === "unsupported" ? 0 : 1); + expect(f.calls.slice(1)).toHaveLength(["live_writer", "older_retained", "unsupported"].includes(state) ? 0 : 1); if (f.calls.length > 1) { expect(call).toHaveBeenCalledExactlyOnceWith(f.pluginId, "workspaceOperation", expect.objectContaining({ executionWorkspaceId: workspaceId, runId: retry.id, operation: expect.objectContaining({ verifyExisting: true }), diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 77abc489223..918385e39f3 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -317,6 +317,7 @@ export function environmentRunOrchestrator( | { kind: "plugin_task"; leaseId: string; preparation?: true }; /** A new controller must stop the previous plugin task before allocating a replacement. */ recoverPluginTask?: boolean; + replacementWorkspaceCreation?: string; }): Promise { // Step 1: Resolve environment const selectedEnvironment = await resolveEnvironment({ @@ -370,6 +371,7 @@ export function environmentRunOrchestrator( executionWorkspaceSettings: input.executionWorkspaceSettings, runnerTaskProjectIds: input.runnerTaskProjectIds, runnerTaskResumeSessionId: input.runnerTaskResumeSessionId, + replacementWorkspaceCreation: input.replacementWorkspaceCreation, adapterType: input.adapterType ?? null, warmRunnerIdleTimeoutMs, runnerReconnectGraceMs, diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index b637afc081f..3c66e95c0a5 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -484,6 +484,7 @@ export interface EnvironmentDriverAcquireInput { executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; runnerTaskProjectIds?: readonly string[]; runnerTaskResumeSessionId?: string | null; + replacementWorkspaceCreation?: string; /** * The harness/adapter type for this run (the agent's adapter). Drivers that * materialize a per-run sandbox use it to select the runtime image so a single @@ -3473,27 +3474,43 @@ function createPluginEnvironmentDriver( }; } - async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) { - await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId))); + async function cleanupPluginRunLease(input: { environment: Environment | null; lease: EnvironmentLease }, + executionStopObserved = false) { const settled = await settleDispatchedEnvironmentTask(db, workerManager, input.lease, { executionStopObserved }); + const [current] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id), + eq(environmentLeases.companyId, input.lease.companyId))); + if (!current || current.providerLeaseId !== input.lease.providerLeaseId) throw new Error("Plugin lease cleanup ownership changed"); + const lease = toEnvironmentLeaseSnapshot(current); + const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease({ ...input, lease }); + const environmentId = input.environment?.id ?? readString(lease.metadata?.environmentId); + if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity"); + const receipt = await workerManager.call(plugin.id, lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", { + driverKey, companyId: lease.companyId, environmentId, issueId: lease.issueId, + config: driverConfig, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata ?? undefined, + }); const taskStopped = settled !== false || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); - if (settled !== "completed") { - const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); - await workerManager.call(plugin.id, "environmentReleaseLease", { - driverKey, - companyId: input.lease.companyId, - environmentId: input.environment.id, - issueId: input.lease.issueId, - config: driverConfig, - providerLeaseId: input.lease.providerLeaseId, - leaseMetadata: input.lease.metadata ?? undefined, - }); + (lease.metadata?.runnerTaskLease === true && lease.metadata?.runnerTaskDispatched !== true && !lease.metadata?.runnerTaskStartup && !lease.metadata?.runnerTaskStartupCleanup); + return remoteTerminationReceipt(lease, receipt) ? receipt + : taskStopped ? { providerLeaseId: lease.providerLeaseId, state: "stopped" } : undefined; + } + + async function releasePluginRunLease(input: EnvironmentDriverReleaseInput, executionStopObserved = false) { + const scope = and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId)); + await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(scope); + let receipt: Awaited>; + try { + receipt = await cleanupPluginRunLease(input, executionStopObserved); + } catch (error) { + await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", + failureReason: "release_cleanup_failed", releasedAt: new Date(), updatedAt: new Date(), + }).where(and(scope, inArray(environmentLeases.status, ["active", "retained"]), + eq(environmentLeases.providerLeaseId, input.lease.providerLeaseId!))), + { companyId: input.lease.companyId, runId: input.lease.heartbeatRunId ?? undefined }); + throw error; } return await environmentsSvc.releaseLease(input.lease.id, input.status, { cleanupStatus: "success", - ...(taskStopped ? { remoteExecutionTermination: remoteTerminationReceipt(input.lease, - { providerLeaseId: input.lease.providerLeaseId, state: "stopped" }) } : {}), + remoteExecutionTermination: remoteTerminationReceipt(input.lease, receipt), }); } @@ -3574,7 +3591,9 @@ function createPluginEnvironmentDriver( companyId: input.companyId, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId, issueId: input.issueId, executionWorkspaceId: input.executionWorkspaceId, provider: `plugin:${parsed.config.pluginKey}:${parsed.config.driverKey}`, pluginId: plugin.id, driverKey: parsed.config.driverKey, - config: {}, timeoutMs: rpcTimeoutMs, requestKey: fingerprint, + config: {}, timeoutMs: rpcTimeoutMs, + requestKey: input.replacementWorkspaceCreation && input.replacementWorkspaceCreation === input.executionWorkspaceId + ? `${fingerprint}:${input.executionWorkspaceId}` : fingerprint, metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: parsed.config.pluginKey, driverKey: parsed.config.driverKey, ...(input.purpose ? { purpose: input.purpose, loginCommandKey: input.loginCommandKey } : {}) }, }, identity => workerManager.call(plugin.id, "environmentAcquireLease", { @@ -3694,18 +3713,7 @@ function createPluginEnvironmentDriver( } catch { return false; } }, - async retryPendingSandboxTeardown(input) { - const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); - const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); - const environmentId = input.environment?.id ?? readString(input.lease.metadata?.environmentId); - if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity"); - const receipt = await workerManager.call(plugin.id, input.lease.metadata?.pluginLeaseCleanupMethod === "destroy" ? "environmentDestroyLease" : "environmentReleaseLease", { - driverKey, companyId: input.lease.companyId, environmentId, issueId: input.lease.issueId, - config: driverConfig, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, - }); - return taskStopped ? { providerLeaseId: input.lease.providerLeaseId, state: "stopped" } : receipt; - }, + retryPendingSandboxTeardown: cleanupPluginRunLease, async realizeWorkspace(input) { const { plugin, pluginKey, driverKey, driverConfig } = await resolvePluginDriverForRelease({ @@ -4045,6 +4053,7 @@ export function environmentRuntimeService( executionWorkspaceSettings?: IssueExecutionWorkspaceSettings | null; runnerTaskProjectIds?: readonly string[]; runnerTaskResumeSessionId?: string | null; + replacementWorkspaceCreation?: string; /** The agent's adapter type for this run (mixed-harness environments). */ adapterType?: string | null; resolveSnapshotIdentity?: boolean; @@ -4091,6 +4100,7 @@ export function environmentRuntimeService( executionWorkspaceSettings: input.executionWorkspaceSettings ?? null, runnerTaskProjectIds: input.runnerTaskProjectIds, runnerTaskResumeSessionId: input.runnerTaskResumeSessionId, + replacementWorkspaceCreation: input.replacementWorkspaceCreation, purpose: input.purpose, loginCommandKey: input.loginCommandKey, adapterType: input.adapterType ?? null, diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index 87020bb274f..f25be7aa4e9 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -565,13 +565,15 @@ export async function executeEnvironmentTask(db: Db, workers: Pick tx.update(environmentLeases).set({ - ...(owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}), + ...(owner.id !== lease.id && owner.status === "retained" ? {status:"released" as const,releasedAt:new Date(),cleanupStatus:"success" as const}:{}), metadata:sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({remoteExecutionTermination:receipt,runnerTaskExecutionStopped:true})}::jsonb`, }).where(and(eq(environmentLeases.id,owner.id),eq(environmentLeases.providerLeaseId,taskId))), { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined }); } } await db.update(environmentLeases).set({ - metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskDispatched":false}'::jsonb`, + metadata: sql`(case when ${environmentLeases.metadata}->>'runnerTaskPreparationStopped' = 'true' + then coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'runnerTaskStartup' + else coalesce(${environmentLeases.metadata}, '{}'::jsonb) end) || '{"runnerTaskDispatched":false}'::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.providerLeaseId, taskId), sql`${environmentLeases.metadata}->>'runnerTaskExecutionStopped' = 'true'`)); } @@ -597,8 +599,11 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskRetiring":true}'::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId))) .returning({ metadata: environmentLeases.metadata, heartbeatRunId: environmentLeases.heartbeatRunId, taskId: environmentLeases.providerLeaseId }); - const startup = current?.metadata?.runnerTaskStartup; const startupCleanup = current?.metadata?.runnerTaskStartupCleanup as { requiresPreparationStop?: boolean } | undefined; + const startup = current?.metadata?.runnerTaskStartup; + if (startupCleanup && current?.metadata?.runnerTaskExecutionStopped === true && current.metadata.runnerTaskDispatched === false && + !current.metadata.runnerTaskAdmissionRejection && !startup && + (!startupCleanup.requiresPreparationStop || current.metadata.runnerTaskPreparationStopped === true)) return "completed"; if (current?.metadata?.runnerTaskDispatched !== true && !startup && !startupCleanup) { if (current?.metadata) await settlePluginTaskCredentialHandoff(db, { companyId: lease.companyId, leaseId: lease.id, expectedTaskId: current.taskId ?? undefined, diff --git a/server/src/services/fresh-workspace-continuation.ts b/server/src/services/fresh-workspace-continuation.ts index 710147a2bfd..5baa71ba0d2 100644 --- a/server/src/services/fresh-workspace-continuation.ts +++ b/server/src/services/fresh-workspace-continuation.ts @@ -20,6 +20,7 @@ import { taskRetryRunAncestry } from "./recovery-run-ancestry.js"; import { parseNativeExecutionInput } from "../vendor/paperclip-runner/index.js"; import { nativeExecutionUsesTaskWorkspace } from "./heartbeat/external-provider.js"; import { preparePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; +import { findWorkspaceProviderFailure } from "./workspace-provider-failure.js"; import { readFreshContinuationReadiness } from "./fresh-continuation-fallback.js"; export type ContinuationReadiness = @@ -522,6 +523,9 @@ export async function verifyRetryWorkspace( issueId: string, responsibleUserId: string | null, readiness: ContinuationReadiness, + options?: { + selectCandidate?: (workspace: typeof executionWorkspaces.$inferSelect) => Promise; + }, ): Promise { if (readiness.mode !== "fresh_isolated_workspace" || run.contextSnapshot?.taskRetryRequestId !== readiness.retryIntentId) return readiness; @@ -541,7 +545,7 @@ export async function verifyRetryWorkspace( companyId: run.companyId, issueId, agentId: run.agentId, successorRunId: run.id, source, cause: null, evidence: null, sourceRetryIntentId: source.contextSnapshot?.taskRetryRequestId, - verifyProviderWorkspace: workers ? async workspace => { + verifyProviderWorkspace: options?.selectCandidate ?? (workers ? async workspace => { try { return Boolean(await preparePluginWorkspace(db, workers, { companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id, responsibleUserId, @@ -549,15 +553,22 @@ export async function verifyRetryWorkspace( existingWorkspaceId: workspace.id, mode: workspace.mode, strategy: workspace.strategyType, verifyExisting: true, })); - } catch { + } catch (error) { + const failure = findWorkspaceProviderFailure(error); + if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error; return false; } - } : undefined, + } : undefined), }); if (!retained) return readiness; const reuse: ContinuationReadiness = { mode: "reuse_issue_workspace", retryIntentId: readiness.retryIntentId, sourceRunId: source.id, workspaceId: retained.id, cwd: retained.cwd }; - await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery", - action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: reuse }); + if (!options?.selectCandidate) await recordVerifiedRetryWorkspace(db, run, issueId, reuse); return reuse; } + +export async function recordVerifiedRetryWorkspace(db: Db, run: { companyId: string; id: string }, issueId: string, + readiness: Extract) { + await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "task-recovery", + action: "issue.retry_workspace_reused", entityType: "issue", entityId: issueId, runId: run.id, details: readiness }); +} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 12de5b29fab..7866f601d3a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -34,7 +34,7 @@ import { registerCancelledWorkspacePreservationWork } from "./native-runtime/nat import { readUpgradeReadiness } from "./heartbeat/upgrade-readiness.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; -import { readFreshWorkspaceReadiness, verifyRetryWorkspace } from "./fresh-workspace-continuation.js"; +import { readFreshWorkspaceReadiness, verifyRetryWorkspace, type ContinuationReadiness } from "./fresh-workspace-continuation.js"; import { settleFailedNativeRecoveryPreparation } from "./native-runtime/native-recovery-preparation-failure.js"; import { promptInjectionPreventionService } from "./prompt-injection-prevention.js"; import { createHeartbeatAgentStatus } from "./heartbeat/agent-status.js"; @@ -49,7 +49,7 @@ import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkin import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral } from "./heartbeat/agent-directory-busy.js"; import { preparePredecessorRetirementFallback } from "./heartbeat/predecessor-retirement-fallback.js"; import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js"; -import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; +import { bindHeartbeatRunnerHomeMode, buildHeartbeatWorkspaceConfiguration, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; import { adoptsLiveRemoteRunner } from "./heartbeat/live-runner-adoption.js"; import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js"; import { watchShutdownHandoff } from "./heartbeat/shutdown-handoff.js"; @@ -60,7 +60,7 @@ import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heart import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; import { reconcileMuseNativeTerminalRun } from "./native-runtime/muse-terminal-reconciliation.js"; -import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; +import { prepareHeartbeatWorkspace, RetryWorkspaceRefused } from "./heartbeat/workspace-preparation.js"; import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js"; import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; @@ -2565,10 +2565,20 @@ export function heartbeatService( // Initialization has persisted the active context, including an explicit // absence of identity inherited from an automatic continuation. responsibleUserId = identityContext.responsibleUserId; - if (issueId && freshWorkspace?.mode === "fresh_isolated_workspace") { - const fallback = freshWorkspace; - freshWorkspace = await timePreparation("workspace_verification", () => - verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, fallback)); + const freshRetryFallback = freshWorkspace?.mode === "fresh_isolated_workspace" ? freshWorkspace : undefined; + let retryWorkspaceCandidate: Extract | undefined; + if (issueId && freshRetryFallback && !externalProvider && + parseObject(context.retryWorkspaceRefusal).intentId !== freshRetryFallback.retryIntentId) { + let verifyManagedWorkspace = false; + const retained = await timePreparation("workspace_verification", () => + verifyRetryWorkspace(db, options.pluginWorkerManager, run, issueId, responsibleUserId, freshRetryFallback, + agent.adapterType === "paperclip_runner" && options.pluginWorkerManager + ? { selectCandidate: async () => { verifyManagedWorkspace = true; return true; } } : undefined)); + if (retained.mode === "reuse_issue_workspace") { + freshWorkspace = retained; + if (verifyManagedWorkspace) + retryWorkspaceCandidate = retained; + } } run = { ...run, @@ -2753,29 +2763,33 @@ export function heartbeatService( legacyUseProjectWorkspace: issueAssigneeOverrides?.useProjectWorkspace ?? null, }); - const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace" - ? "isolated_workspace" : trustPreset.kind === "low_trust_review" && - resolvedExecutionWorkspaceMode === "shared_workspace" - ? "isolated_workspace" - : resolvedExecutionWorkspaceMode; - const issueRef = issueContext - ? { - id: issueContext.id, - identifier: issueContext.identifier, - title: issueContext.title, - status: issueContext.status, - priority: issueContext.priority, - workMode: issueContext.workMode, - conversationAgentId: issueContext.conversationAgentId, - conversationUserId: issueContext.conversationUserId, - reviewPolicy: issueContext.reviewPolicy, - description: issueContext.description, - projectId: issueContext.projectId, - projectWorkspaceId: issueContext.projectWorkspaceId, - executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId, - executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference, - } - : null; + const workspaceInputs = () => { + const requestedExecutionWorkspaceMode = freshWorkspace?.mode === "fresh_isolated_workspace" + ? "isolated_workspace" : trustPreset.kind === "low_trust_review" && + resolvedExecutionWorkspaceMode === "shared_workspace" + ? "isolated_workspace" + : resolvedExecutionWorkspaceMode; + const issueRef = issueContext + ? { + id: issueContext.id, + identifier: issueContext.identifier, + title: issueContext.title, + status: issueContext.status, + priority: issueContext.priority, + workMode: issueContext.workMode, + conversationAgentId: issueContext.conversationAgentId, + conversationUserId: issueContext.conversationUserId, + reviewPolicy: issueContext.reviewPolicy, + description: issueContext.description, + projectId: issueContext.projectId, + projectWorkspaceId: issueContext.projectWorkspaceId, + executionWorkspaceId: freshWorkspace?.mode === "fresh_isolated_workspace" ? null : issueContext.executionWorkspaceId, + executionWorkspacePreference: freshWorkspace?.mode === "fresh_isolated_workspace" ? "create_new" : issueContext.executionWorkspacePreference, + } + : null; + return { requestedExecutionWorkspaceMode, issueRef }; + }; + let { requestedExecutionWorkspaceMode, issueRef } = workspaceInputs(); const storedLedgerScope = parseObject(parseObject(run.usageJson).ledgerScope); const runLedgerScope = Object.keys(storedLedgerScope).length > 0 ? storedLedgerScope @@ -2791,18 +2805,144 @@ export function heartbeatService( policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) }, services: { issuesSvc, getPinnedSkillTestContext }, })); - const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({ - agentConfig: config, - projectPolicy: projectExecutionWorkspacePolicy, - issueSettings: issueExecutionWorkspaceSettings, - mode: requestedExecutionWorkspaceMode, - legacyUseProjectWorkspace: - issueAssigneeOverrides?.useProjectWorkspace ?? null, - adapterConfigOverrides: { - ...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))), - ...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}), + const selectWorkspace = async () => { + const sourceWorkspaceConfig = buildExecutionWorkspaceAdapterConfig({ + agentConfig: config, + projectPolicy: projectExecutionWorkspacePolicy, + issueSettings: issueExecutionWorkspaceSettings, + mode: requestedExecutionWorkspaceMode, + legacyUseProjectWorkspace: + issueAssigneeOverrides?.useProjectWorkspace ?? null, + adapterConfigOverrides: { + ...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))), + ...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}), + }, + }); + const selection = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, { + run, agent, + task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace, + freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId }, + policy: { + trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, + projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, + }, + config: { config, sourceWorkspaceConfig, issueAssigneeOverrides }, + services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder }, + })); + return { ...selection, sourceWorkspaceConfig }; + }; + let workspaceSelection = await selectWorkspace(); + const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { + run, agent, + task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest, + adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) }, + config: { sourceWorkspaceConfig: workspaceSelection.sourceWorkspaceConfig, aiBinding }, + policy: { trustPreset, useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory, + explicitTaskDirectory: workspaceSelection.explicitTaskDirectory, requestedExecutionWorkspaceMode }, + environment: workspaceSelection, + services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral }, + onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; }, + onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; }, + onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; }, + })); + if (!executionConfiguration.prepared) return; + let { mergedConfig, configSnapshot } = executionConfiguration; + const { runScopedMentionedSkillKeys, githubSelection, useHostGitHub, + resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, + runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability, + } = executionConfiguration; + let { runtimeConfig } = executionConfiguration; + const resolvedFailureSecrets = readFailureReportSecrets(); + readFailureReportSecrets = () => [ + ...resolvedFailureSecrets, + ...identityRedactor.values, + ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys), + ]; + const prepareSession = () => timePreparation("session", () => prepareHeartbeatSession(db, { + run, agent, + task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision }, + session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec }, + config: { + runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides, + managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext, }, - }); + workspace: { + ...workspaceSelection, requestedExecutionWorkspaceMode, projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, + }, + environment: workspaceSelection, + getLatestAgentConfigRevision, + })); + let preparedSession = await prepareSession(); + const prepareWorkspace = () => timePreparation("workspace", () => prepareHeartbeatWorkspace(db, { + run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal, + task: { + issueRef, + issueId, + wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, + context, + responsibleUserId, + previousSessionParams: preparedSession.previousSessionParams, + retryWorkspaceCandidate, + workspaceSelection: taskWorkspaceIntentRow?.intent ?? null, + freshWorkspace, + }, + policy: { + trustPreset, + isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, + effectiveExecutionWorkspaceMode: workspaceSelection.effectiveExecutionWorkspaceMode, + requestedExecutionWorkspaceMode, + useIsolatedTaskDirectory: workspaceSelection.useIsolatedTaskDirectory, + nativeChatWorkspaceScope: workspaceSelection.nativeChatWorkspaceScope, + projectExecutionWorkspacePolicy, + issueExecutionWorkspaceSettings, + environmentExecutionWorkspaceSettings, + executionPolicy: workspaceSelection.executionPolicy, + }, + environment: workspaceSelection, + config: { + mergedConfig, + configSnapshot, + secretManifest, + managedAiCredentialIdentity: managedAiRuntime?.identity, + runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId, + }, + reuse: { + ...workspaceSelection, + runOptions, + }, + services: { + workspaceWorkers: options.pluginWorkerManager, + envOrchestrator, + executionWorkspacesSvc, + workspaceOperationsSvc, + resolveWorkspaceForRun, + resolveReusedGitWorkspaceAnchor, + appendRunEvent, + }, + })); + let preparedWorkspace; + try { + preparedWorkspace = await prepareWorkspace(); + } catch (error) { + if (!(error instanceof RetryWorkspaceRefused) || !freshRetryFallback) throw error; + retryWorkspaceCandidate = undefined; + freshWorkspace = freshRetryFallback; + ({ requestedExecutionWorkspaceMode, issueRef } = workspaceInputs()); + const selected = await selectWorkspace(); + if (selected.selectedEnvironmentId !== workspaceSelection.selectedEnvironmentId) + throw new Error("Retry environment changed during workspace verification"); + workspaceSelection = selected; + ({ mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration( + { ...selected.sourceWorkspaceConfig, homeMode: mergedConfig.homeMode }, + { trustPreset, useIsolatedTaskDirectory: selected.useIsolatedTaskDirectory, + explicitTaskDirectory: selected.explicitTaskDirectory, requestedExecutionWorkspaceMode }, + selected.selectedEnvironmentId)); + runtimeConfig = { ...runtimeConfig }; + if (Object.hasOwn(mergedConfig, "workspaceStrategy")) runtimeConfig.workspaceStrategy = mergedConfig.workspaceStrategy; + else delete runtimeConfig.workspaceStrategy; + preparedSession = await prepareSession(); + preparedWorkspace = await prepareWorkspace(); + } const { persistedRunnerProfile, persistedNativeExecutionInput, @@ -2825,58 +2965,9 @@ export function heartbeatService( materializeNativeChatWorktree, explicitTaskDirectory, useIsolatedTaskDirectory, - } = await timePreparation("environment_selection", () => selectHeartbeatEnvironment(db, { - run, agent, - task: { issueRef, issueContext, issueId, context, executionProjectId, workspaceProjectContext, boundSourceWorkspace, - freshWorkspace, hasSelectedWorkspaceSource: Boolean(selectedWorkspaceSource), taskWorkspaceIntent: taskWorkspaceIntentRow?.intent ?? null, responsibleUserId }, - policy: { - trustPreset, requestedExecutionWorkspaceMode, isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, - projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, - }, - config: { config, sourceWorkspaceConfig, issueAssigneeOverrides }, - services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder }, - })); + sourceWorkspaceConfig, + } = workspaceSelection; let nativeChatExpectedCwd = selectedNativeChatExpectedCwd; - const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { - run, agent, - task: { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, taskProjectContext, routineEnvContext, pinnedSkillTest, - adoptingLiveRunner: adoptsLiveRemoteRunner({ run, restartRecovery: runOptions.nativeRestartRecovery, leaseOwner: runOptions.nativeLeaseOwner }) }, - config: { sourceWorkspaceConfig, aiBinding }, - policy: { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }, - environment: { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput }, - services: { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral }, - onFailureSecretsResolved: (readSecrets) => { readFailureReportSecrets = () => [...readSecrets(), ...identityRedactor.values]; }, - onManagedAiRuntimeAcquired: (runtime) => { managedAiRuntime = runtime; }, - onIdentityRedactorPrepared: (redactor) => { identityRedactor = redactor; }, - })); - if (!executionConfiguration.prepared) return; - const { - mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub, - resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest, - runtimeSkillEntries, nativeRunnerPreparationSpans, agentIdentity, observability, - } = executionConfiguration; - let { runtimeConfig } = executionConfiguration; - const resolvedFailureSecrets = readFailureReportSecrets(); - readFailureReportSecrets = () => [ - ...resolvedFailureSecrets, - ...identityRedactor.values, - ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys), - ]; - const preparedSession = await timePreparation("session", () => prepareHeartbeatSession(db, { - run, agent, - task: { context, issueRef, issueContext, taskProjectContext, workspaceProjectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision }, - session: { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec }, - config: { - runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides, - managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext, - }, - workspace: { - requestedExecutionWorkspaceMode, effectiveExecutionWorkspaceMode, projectExecutionWorkspacePolicy, - issueExecutionWorkspaceSettings, executionPolicy, requestedReusableExecutionWorkspaceConfig, reusableExistingExecutionWorkspace, - }, - environment: { environmentResolution, selectedEnvironmentId, selectedEnvironmentForConfig }, - getLatestAgentConfigRevision, - })); const { sessionConfigMetadata, configuredModel, resetTaskSession, sessionResetReason, taskSessionForRun, getFreshSessionHandoff, previousSessionParams } = preparedSession; const { resolvedWorkspace, @@ -2898,64 +2989,7 @@ export function heartbeatService( workspaceRealization, executionTarget, workspaceRestoreSource, - } = await timePreparation("workspace", () => prepareHeartbeatWorkspace(db, { - run, agent, controllerLease, nativeRunnerPreparationSpans, signal: executionControl.controller.signal, - task: { - issueRef, - issueId, - wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, - context, - responsibleUserId, - previousSessionParams, - workspaceSelection: taskWorkspaceIntentRow?.intent ?? null, - freshWorkspace, - }, - policy: { - trustPreset, - isolatedWorkspacesEnabled: runtimeWorkspaceSelectionEnabled, - effectiveExecutionWorkspaceMode, - requestedExecutionWorkspaceMode, - useIsolatedTaskDirectory, - nativeChatWorkspaceScope, - projectExecutionWorkspacePolicy, - issueExecutionWorkspaceSettings, - environmentExecutionWorkspaceSettings, - executionPolicy, - }, - environment: { - selectedEnvironmentId, - localEnvironment, - selectedEnvironmentForConfig, - environmentResolution, - resolvedInstanceSettings, - }, - config: { - mergedConfig, - configSnapshot, - secretManifest, - managedAiCredentialIdentity: managedAiRuntime?.identity, - runnerTaskResumeSessionId: run.nativeSessionId ?? preparedSession.taskResumeSessionId, - }, - reuse: { - requestedShouldReuseExisting, - existingExecutionWorkspace, - reusableExistingExecutionWorkspace, - workspaceReuseRequest, - nativeRecoveryExecutionWorkspaceId, - persistedNativeExecutionInput, - isDotRun, - runOptions, - }, - services: { - workspaceWorkers: options.pluginWorkerManager, - envOrchestrator, - executionWorkspacesSvc, - workspaceOperationsSvc, - resolveWorkspaceForRun, - resolveReusedGitWorkspaceAnchor, - appendRunEvent, - }, - })); + } = preparedWorkspace; const preparedInstructions = instructionWorkingCopy = createHeartbeatInstructionWorkingCopy({ observabilityCompatibilityKey: observability?.compatibilityKey, db, instructionCopies, readRun: () => run!, agent, context, diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 56fc9ca1a06..248cd400687 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -94,6 +94,23 @@ export interface HeartbeatExecutionConfigurationInput { onIdentityRedactorPrepared: (redactor: ReturnType) => void; } +export function buildHeartbeatWorkspaceConfiguration( + sourceWorkspaceConfig: Record, + { useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode }: HeartbeatExecutionConfigurationInput["policy"], + selectedEnvironmentId: string | null, +) { + const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && ( + requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0 + )); + const mergedConfig: Record = { + ...sourceWorkspaceConfig, + // The base below is already task-owned. Keep directory transport while + // preserving isolated mode and the mandatory sandbox preflight. + ...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}), + }; + return { mergedConfig, configSnapshot: buildExecutionWorkspaceConfigSnapshot(mergedConfig, selectedEnvironmentId) }; +} + /** Assembles effective run configuration; the executor retains cleanup ownership. */ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: HeartbeatExecutionConfigurationInput) { const { run, agent, onFailureSecretsResolved, onManagedAiRuntimeAcquired, onIdentityRedactorPrepared } = input; @@ -137,22 +154,10 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear // preparation step can fail. Subsequent provider/identity values extend it. onFailureSecretsResolved(() => []); const aiBinding = preconfiguredHome ? undefined : input.config.aiBinding; - const { trustPreset, useIsolatedTaskDirectory, explicitTaskDirectory, requestedExecutionWorkspaceMode } = input.policy; + const { trustPreset } = input.policy; const { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral } = input.services; let managedAiRuntime: ManagedAiRuntime | undefined; - const forceDirectoryStrategy = useIsolatedTaskDirectory || (explicitTaskDirectory && ( - requestedExecutionWorkspaceMode === "agent_default" || Object.keys(parseObject(sourceWorkspaceConfig.workspaceStrategy)).length > 0 - )); - const mergedConfig = { - ...sourceWorkspaceConfig, - // The base below is already task-owned. Keep directory transport while - // preserving isolated mode and the mandatory sandbox preflight. - ...(forceDirectoryStrategy ? { workspaceStrategy: { type: "project_primary" } } : {}), - }; - const configSnapshot = buildExecutionWorkspaceConfigSnapshot( - mergedConfig, - selectedEnvironmentId, - ); + const { mergedConfig, configSnapshot } = buildHeartbeatWorkspaceConfiguration(sourceWorkspaceConfig, input.policy, selectedEnvironmentId); const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index bab1110db5a..c9d5bd9224e 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1797,7 +1797,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // path uses the provider and configuration recorded on the lease first; // the environment is lifecycle context and only a legacy fallback. const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral"; - const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease); + const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease) || + (metadata.driver === "plugin" && metadata.pluginLeaseCleanupMethod === "release"); // Do not consume a finite cleanup attempt while the provider plugin is // briefly unavailable. A plugin worker restart, a plugin reload, or a diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 06b3ae06a1e..d2039d644f3 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -1,3 +1,4 @@ +import { preparePluginWorkspace } from "../plugin-workspaces.js"; import { execFile as execFileCallback } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdtemp, mkdir, readFile, readdir, rm, writeFile, access, rename, symlink } from "node:fs/promises"; @@ -23,7 +24,7 @@ import { instanceSettingsService } from "../instance-settings.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { nativeWorkspaceSyncInternals, prepareNativeWorkspaceSync } from "../native-runtime/native-workspace-sync.js"; import { createHeartbeatWorkspaceResolver, prepareProjectRepositoryWorkspaces, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; -import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js"; +import { RetryWorkspaceRefused, prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js"; import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; import * as managedRepositoryCheckout from "../managed-repository-checkout.js"; @@ -965,15 +966,18 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replace: true }, { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replaceAfterPrepare: true }, - ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare }) => { + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, refuse: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, verify: true, restart: true }, + ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare, verify: $verify, refuse: $refuse, restart: $restart)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare, verify, refuse, restart }) => { const { input, source, issue } = await fixture(true); if (portable) input.agent.adapterType = "paperclip_runner"; - const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : ""}`; + const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : refuse ? "-refuse" : restart ? "-restart" : verify ? "-verify" : ""}`; const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: "test-storage", version: "1.0.0", apiVersion: 1, status: "ready", categories: ["workspace"], manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Storage", description: "Storage", author: "Test", categories: ["workspace"], - workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], + workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, workspaceVerificationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" }, - ...(portable ? { sessionHandoffVersion: 1 as const } : {}) }], + ...(portable ? { sessionHandoffVersion: 1 as const, freshWorkspaceVersion: 1 as const } : {}) }], resourceLifecycle: ["project"], entrypoints: { worker: "./worker.js" } } }).returning(); const [environment] = await db.insert(environments).values({ name: `Storage tasks ${pluginKey}`, driver: "plugin", status: "active", config: { pluginKey, driverKey: "tasks", driverConfig: {} } }).returning(); @@ -1001,15 +1005,20 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace const prepareRequests: Array<{ operation: Parameters[2]["operation"]; operationId: string }> = []; const finalizationIds: string[] = []; let executionWorkspaceId: string | undefined; - input.services.workspaceWorkers = { getWorker: () => ({ supportedMethods: [ - "workspaceOperation", "environmentTask", ...(portable ? ["workspacePrepare"] : []), + let loseVerificationReply = restart === true; + input.services.workspaceWorkers = { isRunning: () => true, getWorker: () => ({ supportedMethods: [ + "workspaceOperation", "environmentTask", "environmentReleaseLease", ...(portable ? ["workspacePrepare"] : []), ] }), call: async (_: string, method: string, params: any) => { if (method === "environmentAcquireLease") { - expect(replace).toBe(true); + expect(replace || restart).toBe(true); expect(params.executionWorkspaceId).toBe(executionWorkspaceId); - sequence.push("replace"); + sequence.push(restart ? "allocate" : "replace"); return { providerLeaseId: readyTaskId }; } + if (method === "environmentReleaseLease") { + sequence.push("release"); + return { providerLeaseId: params.providerLeaseId, state: "stopped" }; + } if (method === "environmentTask") { if (replace && params.operation.kind !== "describe") { expect(params.taskId).toBe(providerLeaseId); @@ -1023,28 +1032,49 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace sequence.push("placement"); return { kind: "description", taskId: readyTaskId, runner: { version: "test", digest: `sha256:${"a".repeat(64)}` } }; } - expect(params.taskLease).toEqual(portable ? taskLease : undefined); + expect(params.taskLease).toEqual(portable && (!verify || executionWorkspaceId) ? taskLease : undefined); if (params.operation.kind === "finalize") { sequence.push("finalize"); finalizationIds.push(params.operationId); return { kind: "finalized", headSha: "a".repeat(40), branchName: "test-WSP-1" }; } - if (portable) { + if (params.operation.verifyExisting) { + expect(method).toBe("workspaceOperation"); + sequence.push("workspace verify"); + if (loseVerificationReply) { + loseVerificationReply = false; + throw new Error("workspace verification reply lost"); + } + } else if (portable && (!verify || executionWorkspaceId)) { expect(method).toBe("workspacePrepare"); expect(params.phase).toBe("submit"); expect(params.executionWorkspaceId).toBe(executionWorkspaceId); prepareRequests.push(params); } - sequence.push("workspace prepare"); + if (!params.operation.verifyExisting) sequence.push("workspace prepare"); if (replaceAfterPrepare && prepareRequests.length === 2) throw new Error("lost workspace preparation reply"); return { kind: "prepared", binding: { pluginId: plugin.id, projectId: issue.projectId, executionWorkspaceId: params.executionWorkspaceId, projectWorkspaceId: issue.projectWorkspaceId, sourceRevision: params.operation.source.revision, workspaceRef: replaceAfterPrepare && prepareRequests.length === 3 ? "provider:replacement-checkout" : "provider:checkout" }, - branchName: params.operation.branchName, baseSha: "a".repeat(40), created: true, branchCreatedByRuntime: true }; + branchName: params.operation.branchName, baseSha: "a".repeat(40), created: refuse || !params.operation.verifyExisting, branchCreatedByRuntime: true }; } } as any; input.services.envOrchestrator = environmentRunOrchestrator(db, { pluginWorkerManager: input.services.workspaceWorkers as never, }); + if (verify) { + const prepared = await preparePluginWorkspace(db, input.services.workspaceWorkers, { + companyId: input.agent.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, + issueId: issue.id, runId: input.run.id, agentId: input.agent.id, responsibleUserId: null, + existingWorkspaceId: null, mode: "isolated_workspace", strategy: "git_worktree", branchName: "test-WSP-1", + }); + const workspace = (await input.services.executionWorkspacesSvc.getStoredById(input.agent.companyId, prepared!.id))!; + input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id }; + input.reuse.existingExecutionWorkspace = input.reuse.reusableExistingExecutionWorkspace = workspace; + input.task.freshWorkspace = input.task.retryWorkspaceCandidate = { mode: "reuse_issue_workspace", + retryIntentId: randomUUID(), sourceRunId: randomUUID(), workspaceId: workspace.id, cwd: workspace.cwd }; + sequence.length = 0; + prepareRequests.length = 0; + } const acquire = input.services.envOrchestrator.acquireForRun; vi.spyOn(input.services.envOrchestrator, "acquireForRun").mockImplementation(async args => { sequence.push("acquire"); @@ -1054,6 +1084,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace } expect(args.persistedExecutionWorkspace).toMatchObject({ mode: defaultMode, id: expect.any(String) }); executionWorkspaceId = args.persistedExecutionWorkspace!.id; + if (restart) return acquire(args); const [lease] = await db.insert(environmentLeases).values({ companyId: args.companyId, environmentId: environment.id, heartbeatRunId: args.heartbeatRunId, issueId: args.issueId, executionWorkspaceId: args.persistedExecutionWorkspace!.id, provider: `plugin:${pluginKey}:tasks`, providerLeaseId, @@ -1089,7 +1120,50 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace } throw new Error("reached execution handoff"); }); - const portableSequence = ["acquire", ...(replace ? ["replace"] : []), "placement", "workspace prepare"]; + if (restart) { + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ kind: "transport", retryable: true }); + expect(sequence).toEqual(["acquire", "allocate", "workspace verify"]); + const [lease] = await leases(input.run.id); + expect(lease).toMatchObject({ providerLeaseId, status: "active" }); + expect((await readRun(input.run.id)).contextSnapshot?.retryWorkspaceRefusal).toBeUndefined(); + input.reuse.runOptions.nativeRestartRecovery = { + kind: "bootstrap_incomplete", runId: input.run.id, leaseOwner: "restarted-controller", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + }; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff"); + expect(sequence).toEqual(["acquire", "allocate", "workspace verify", "acquire", "workspace verify", "placement", "workspace prepare"]); + expect((await leases(input.run.id)).map(item => item.id)).toEqual([lease.id]); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskDispatched: true } }) + .where(eq(environmentLeases.id, lease.id)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("Workspace verification lease already started execution"); + expect(sequence.slice(-2)).toEqual(["workspace prepare", "acquire"]); + return; + } + if (refuse) { + const candidate = input.task.retryWorkspaceCandidate!; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow(RetryWorkspaceRefused); + expect(sequence).toEqual(["acquire", "workspace verify", "release"]); + const persisted = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.run.id)).then(([row]) => row); + expect(persisted.contextSnapshot?.retryWorkspaceRefusal).toMatchObject({ + intentId: candidate.retryIntentId, workspaceId: candidate.workspaceId, leaseId: expect.any(String), + }); + expect((await leases(input.run.id))[0]).toMatchObject({ status: "released", cleanupStatus: "success" }); + const next = new Error("fresh acquisition reached"); + vi.mocked(input.services.envOrchestrator.acquireForRun).mockRejectedValue(next); + await expect(prepareHeartbeatWorkspace(db, { + ...input, run: persisted, + reuse: { ...input.reuse, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null }, + task: { ...input.task, context: persisted.contextSnapshot!, retryWorkspaceCandidate: undefined, + issueRef: { ...input.task.issueRef!, executionWorkspaceId: null }, + freshWorkspace: { mode: "fresh_isolated_workspace", retryIntentId: candidate.retryIntentId, + sourceRunId: candidate.sourceRunId, retainedSourceIds: [] } }, + })).rejects.toBe(next); + expect(input.services.envOrchestrator.acquireForRun).toHaveBeenLastCalledWith(expect.objectContaining({ + replacementWorkspaceCreation: expect.any(String), persistedExecutionWorkspace: expect.objectContaining({ id: expect.not.stringMatching(candidate.workspaceId) }), + })); + return; + } + const portableSequence = ["acquire", ...(verify ? ["workspace verify"] : []), ...(replace ? ["replace"] : []), "placement", "workspace prepare"]; if (replaceAfterPrepare) { const { executionTarget } = await prepareHeartbeatWorkspace(db, input); if (executionTarget?.kind !== "remote" || executionTarget.transport !== "sandbox" || !executionTarget.runnerTask) diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index d437a8cc273..99403f393b7 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -5,7 +5,8 @@ import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, Remote import { WorkspaceBusyDeferral } from "./retries.js"; import { remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold.js"; import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js"; -import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js"; +import { findWorkspaceProviderFailure } from "../workspace-provider-failure.js"; +import { readFreshWorkspaceReadiness, recordVerifiedRetryWorkspace, type ContinuationReadiness } from "../fresh-workspace-continuation.js"; import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js"; import { NativeExecutionContext } from "../native-runtime/native-execution-context.js"; import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js"; @@ -39,9 +40,9 @@ import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import fs from "node:fs/promises"; import path from "node:path"; import { CONTROLLER_PROVISION_COPY_ONLY, workspaceRequiresCopyOnlyProvisioning } from "../workspace-host-provision.js"; -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, sql } from "drizzle-orm"; import { type ExecutionWorkspace } from "@paperclipai/shared"; -import { executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db"; +import { environmentLeases, executionWorkspaces as executionWorkspacesTable, heartbeatRuns, issues, projectWorkspaces } from "@paperclipai/db"; import { getStartupTracer } from "../../instrumentation.js"; import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js"; import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js"; @@ -103,6 +104,8 @@ type WorkspaceMetadataInput = Parameters | null; freshWorkspace?: import("../fresh-workspace-continuation.js").ContinuationReadiness | null; + retryWorkspaceCandidate?: Extract; workspaceSelection?: import("@paperclipai/shared").TaskWorkspaceIntent | null; }; policy: { @@ -161,7 +165,7 @@ export interface HeartbeatWorkspacePreparationInput { }; services: { workspaceWorkers?: WorkspaceWorkers; - envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun">; + envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun" | "releaseForRun">; executionWorkspacesSvc: Pick, "create" | "update" | "getStoredById" | "bindTaskWorkspace" | "prepareTaskRepositoriesForAdmission" | "assertTaskRepositoryPublicationAllowed" | "prepareConfiguredRepositoryCopy" | "assertTaskRepositoryRecoveryBinding">; workspaceOperationsSvc: Pick, "createRecorder">; resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"]; @@ -580,6 +584,29 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace throw new Error(allowlistDecision.reason); } }; + const refused = parseObject(context.retryWorkspaceRefusal); + const refusedWorkspaceId = fresh && input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && + refused.intentId === input.task.freshWorkspace.retryIntentId && typeof refused.workspaceId === "string" + ? refused.workspaceId : undefined; + const priorVerificationLease = refusedWorkspaceId && selectedEnvironmentForConfig && typeof refused.leaseId === "string" + ? (await db.select().from(environmentLeases).where(and( + eq(environmentLeases.id, refused.leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.environmentId, selectedEnvironmentForConfig.id), eq(environmentLeases.executionWorkspaceId, refusedWorkspaceId), + )))[0] : undefined; + if (refusedWorkspaceId && typeof refused.leaseId === "string" && !priorVerificationLease) + throw new Error("Workspace verification lease changed"); + const releaseVerificationLease = async (leaseId: string) => { + await controllerLease.assertOwned(); + await envOrchestrator.releaseForRun({ companyId: agent.companyId, agentId: agent.id, heartbeatRunId: run.id }); + const [lease] = await db.select({ status: environmentLeases.status, releasedAt: environmentLeases.releasedAt, + cleanupStatus: environmentLeases.cleanupStatus }).from(environmentLeases).where(and( + eq(environmentLeases.id, leaseId), eq(environmentLeases.companyId, agent.companyId), eq(environmentLeases.heartbeatRunId, run.id))); + if (!lease?.releasedAt || !["released", "expired"].includes(lease.status) || lease.cleanupStatus !== "success") + throw new Error("Workspace verification lease cleanup is pending"); + }; + if (priorVerificationLease && (!priorVerificationLease.releasedAt || priorVerificationLease.cleanupStatus !== "success")) + await releaseVerificationLease(priorVerificationLease.id); + const retryCandidate = input.task.retryWorkspaceCandidate; const environmentAcquisitionInput = ( persistedExecutionWorkspace: Parameters[0]["persistedExecutionWorkspace"], projectId: string | null, @@ -596,46 +623,64 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace runnerTaskProjectIds: [...new Set([projectId, ...resolvedWorkspace.runnerTaskProjectIds ?? [], ...(additionalWorkspaces ?? []).map(workspace => workspace.projectId)] .filter((id): id is string => Boolean(id)))], - recoverPluginTask: !portablePreparationLeaseId && + recoverPluginTask: !priorVerificationLease && !portablePreparationLeaseId && + !(retryCandidate && restartRecovery?.kind === "bootstrap_incomplete") && (restartRecovery?.kind === "resume_dead_runner" || restartRecovery?.kind === "bootstrap_incomplete"), issueId: issueId ?? null, heartbeatRunId: run.id, agentId: agent.id, persistedExecutionWorkspace, + ...(priorVerificationLease && persistedExecutionWorkspace && persistedExecutionWorkspace.id !== refusedWorkspaceId + ? { replacementWorkspaceCreation: persistedExecutionWorkspace.id } : {}), executionWorkspaceSettings: environmentExecutionWorkspaceSettings, }); const pluginRecovery = restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.remote.kind === "plugin_task" - ? restartRecovery.remote : portablePreparationLeaseId + ? restartRecovery.remote : portablePreparationLeaseId && portablePreparationLeaseId !== priorVerificationLease?.id ? { kind: "plugin_task" as const, leaseId: portablePreparationLeaseId, preparation: true as const } : undefined; let workspaceAcquisition: Awaited> | undefined; let workspaceAcquireStartedAtMs: number | undefined; let ensureTaskWorkspace: ((taskId: string, assertOwned: () => void) => Promise | undefined) | undefined; - const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, { + const acquireTaskLease: NonNullable[2]["acquireTaskLease"]> = async workspace => { + if (workspaceAcquisition) { + const { environment, lease } = workspaceAcquisition; + if (lease.executionWorkspaceId !== workspace.id || typeof lease.metadata?.driverKey !== "string" || !lease.providerLeaseId) + throw new Error("Workspace placement changed"); + return { leaseId: lease.id, taskLease: { environmentId: environment.id, driverKey: lease.metadata.driverKey, providerLeaseId: lease.providerLeaseId } }; + } + if (!selectedEnvironmentForConfig) throw new Error("Workspace placement is unavailable"); + assertAllowedEnvironment(selectedEnvironmentForConfig); + await controllerLease.assertOwned(); + workspaceAcquireStartedAtMs = Date.now(); + if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id) + throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id"); + const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ + ...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces), + ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}), + })); + await controllerLease.assertOwned(); + if (retryCandidate) { + const metadata = acquisition.lease.metadata; + if (metadata?.runnerTaskDispatched === true || metadata?.runnerTaskIdentity || + metadata?.runnerTaskStartup || metadata?.runnerTaskStartupCleanup || metadata?.runnerTaskRetiring === true) + throw new Error("Workspace verification lease already started execution"); + workspaceAcquisition = acquisition; + return acquireTaskLease(workspace); + } + const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, { + companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal, + }); + await controllerLease.assertOwned(); + workspaceAcquisition = placement.acquisition; + return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease }; + }; + const pluginPrepareInput: Parameters[2] = { companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId, issueId, runId: run.id, agentId: agent.id, responsibleUserId, existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null, ...(providerFresh ? { fresh: providerFresh, expectedBindingRevision: previousProviderBindingRevision } : {}), selectedProviderPluginId: projectExecutionWorkspacePolicy?.storageProviderPluginId ?? null, environment: selectedEnvironmentForConfig, - acquireTaskLease: async workspace => { - if (!selectedEnvironmentForConfig) throw new Error("Workspace Launcher placement is unavailable"); - assertAllowedEnvironment(selectedEnvironmentForConfig); - await controllerLease.assertOwned(); - workspaceAcquireStartedAtMs = Date.now(); - if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id) - throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id"); - const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ - ...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces), - ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}), - })); - await controllerLease.assertOwned(); - const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, { - companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal, - }); - await controllerLease.assertOwned(); - workspaceAcquisition = placement.acquisition; - return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease }; - }, + acquireTaskLease, onTaskPrepared: ensure => { ensureTaskWorkspace = ensure; }, mode: requestedExecutionWorkspaceMode, strategy: resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig), @@ -647,7 +692,45 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace })) : null, baseRef: typeof parseObject(mergedConfig.workspaceStrategy).baseRef === "string" ? String(parseObject(mergedConfig.workspaceStrategy).baseRef) : null, - }); + }; + if (retryCandidate && !controllerOnly && !remoteAuthority) { + const verified = await timeWorkspace("workspace_verification", () => preparePluginWorkspace(db, input.services.workspaceWorkers, { + ...pluginPrepareInput, + existingWorkspaceId: retryCandidate.workspaceId, + verifyExisting: true, + assertOwned: () => input.signal?.throwIfAborted(), + onTaskPrepared: undefined, + })).catch(error => { + const failure = findWorkspaceProviderFailure(error); + if (failure?.reason !== "binding_mismatch" && failure?.reason !== "unsupported_operation") throw error; + return null; + }); + await controllerLease.assertOwned(); + if (!verified) { + const refusal = { intentId: retryCandidate.retryIntentId, workspaceId: retryCandidate.workspaceId, + leaseId: workspaceAcquisition?.lease.id ?? null }; + const [saved] = await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) + || ${JSON.stringify({ retryWorkspaceRefusal: refusal })}::jsonb` }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.status, "running"))) + .returning({ id: heartbeatRuns.id }); + if (!saved) throw new Error("Workspace verification lost run ownership"); + context.retryWorkspaceRefusal = refusal; + if (workspaceAcquisition) await releaseVerificationLease(workspaceAcquisition.lease.id); + throw new RetryWorkspaceRefused(); + } + await recordVerifiedRetryWorkspace(db, run, issueId!, retryCandidate); + if (workspaceAcquisition && selectedEnvironmentForConfig) { + const providerLeaseId = workspaceAcquisition.lease.providerLeaseId; + const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(workspaceAcquisition, { + companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: retryCandidate.workspaceId, signal: input.signal, + }); + if (placement.acquisition.lease.providerLeaseId !== providerLeaseId) + throw new Error("Workspace placement changed after verification"); + workspaceAcquisition = placement.acquisition; + } + } + const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, pluginPrepareInput); + const freshTaskReservationId = providerFresh && pluginTaskWorkspace && !pluginPrepared && issueId ? await reserveFreshPluginTaskWorkspace({ db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id, fresh: providerFresh, responsibleUserId, expectedBindingRevision: previousProviderBindingRevision, owner: taskWorkspaceOwner!, diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index 67c9008b628..fa8efcbc9f9 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -86,6 +86,7 @@ export async function preparePluginWorkspace( fresh?: PluginFreshWorkspace; expectedBindingRevision?: number; verifyExisting?: true; + assertOwned?: () => void; acquireTaskLease?: (workspace: Pick) => Promise<{ leaseId: string; taskLease: TaskPlacement }>; onTaskPrepared?: (ensure: EnsureTaskWorkspace) => void; }, @@ -165,6 +166,7 @@ export async function preparePluginWorkspace( const selectedProvider = await requireWorkspaceProviderWorker(db, workers, input.companyId, pluginId, true); if (input.environment) assertProjectStorageEnvironment(selectedProvider, input.environment); const declaration = selectedProvider.manifestJson.workspaceProvider!; + if (input.verifyExisting && declaration.workspaceVerificationVersion !== 1) return null; if (input.fresh && declaration.freshWorkspaceVersion !== 1) throw conflict("A fresh attempt is unavailable for this workspace storage provider. The existing workspace is retained.", { reason: "fresh_workspace_storage_unsupported" }); if ( @@ -348,7 +350,7 @@ export async function preparePluginWorkspace( driver.driverKey === driverKey && driver.sessionHandoffVersion === 1) ? await input.acquireTaskLease?.({ id: workspace.id, mode: mode as ExecutionWorkspace["mode"] }) : undefined; const prepare = (workspace: Workspace, taskLease?: TaskPlacement, replacement?: { leaseId: string; assertOwned: () => void }) => withExecutionWorkspaceLifecycleLock(db, workspace.id, async (tx) => { - replacement?.assertOwned(); + (replacement?.assertOwned ?? input.assertOwned)?.(); const [current] = await tx .select() .from(executionWorkspaces) @@ -401,7 +403,7 @@ export async function preparePluginWorkspace( runId: input.runId, agentId: input.agentId, taskLease, - assertOwned: replacement?.assertOwned, + assertOwned: replacement?.assertOwned ?? input.assertOwned, operation: { kind: "prepare", source: sourceInput, @@ -430,8 +432,9 @@ export async function preparePluginWorkspace( (saved && result.binding.workspaceRef !== saved.workspaceRef) || (workspace.branchName !== null && result.branchName !== workspace.branchName) || (typeof savedBase === "string" && result.baseSha !== savedBase))) { - throw conflict("The existing checkout could not be verified"); + return null; } + (replacement?.assertOwned ?? input.assertOwned)?.(); await authorize(); const [currentProject] = await tx .select() @@ -485,12 +488,13 @@ export async function preparePluginWorkspace( return updated!; }); const prepared = await prepare(workspace, placement?.taskLease); - if (placement && input.onTaskPrepared) { + if (prepared && placement && input.onTaskPrepared) { let preparedTaskId = placement.taskLease.providerLeaseId; let preparedWorkspace = prepared; input.onTaskPrepared((taskId, assertOwned) => { if (taskId === preparedTaskId) return; return prepare(preparedWorkspace, { ...placement.taskLease, providerLeaseId: taskId }, { leaseId: placement.leaseId, assertOwned }).then(updated => { + if (!updated) throw conflict("Workspace preparation is incomplete"); preparedWorkspace = updated; preparedTaskId = taskId; });