diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index ee582ef6f6d..4ca856513cd 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -172,6 +172,25 @@ It does not qualify a deployed plugin, live account, or production WSS proxy. ## Workspace completion and maintenance +When the selected task driver declares `sessionHandoffVersion: 1`, Core +reserves the managed workspace before it acquires the run's task lease. Core +reconciles placement ownership, then passes `taskLease` to the workspace +preparation call. The plugin uses the same allocation for checkout and Runner +admission. Finalization records that placement before calling the plugin and +reuses it on retry. If an interrupted finalization has no recorded placement, +Core retries its original request without placement. This preserves the +operation ID and the provider's established routing for requests made before +this upgrade. + +Before dispatching preparation for an unbound workspace, Core records the +run and agent, plus the task placement when present. Inspection replays the +saved request. A retry with the same run, agent, and placement uses the same +operation ID. When a retry acquires a different task, Core records its +placement and prepares the same workspace. The provider settles the prior +preparation before using the retained checkout. Reservation reconciliation +uses project Git consent while keeping the original run and agent in the +provider request. + A portable restore uses the durable ownership claim and the same account identity after a credential refresh or controller replacement. It starts a new Runner on the target. Run attachment applies the current tools and diff --git a/packages/plugins/sdk/src/workspace-provider.ts b/packages/plugins/sdk/src/workspace-provider.ts index dea90b92e5b..bdfe5e91d45 100644 --- a/packages/plugins/sdk/src/workspace-provider.ts +++ b/packages/plugins/sdk/src/workspace-provider.ts @@ -119,6 +119,12 @@ export const pluginWorkspaceParamsSchema = z deadline: z.string().datetime(), runId: z.string().uuid().nullable(), agentId: z.string().uuid().nullable(), + /** Placement selected by Core for workspace work on the run's Launcher. */ + taskLease: z.object({ + environmentId: z.string().uuid(), + driverKey: z.string().min(1).max(128), + providerLeaseId: z.string().min(1).max(1024), + }).strict().optional(), operation: workspaceProviderOperationSchema, }) .strict(); diff --git a/server/src/__tests__/fixtures/plugin-worker-invocation-scope.cjs b/server/src/__tests__/fixtures/plugin-worker-invocation-scope.cjs index f01e0a0d701..9462fadfa28 100644 --- a/server/src/__tests__/fixtures/plugin-worker-invocation-scope.cjs +++ b/server/src/__tests__/fixtures/plugin-worker-invocation-scope.cjs @@ -22,6 +22,12 @@ function sendNestedHostRequest(originalRequest, invocationId) { }, configPath: params.configPath || "apiKeyRef", } + : hostMethod === "projects.resolveGitCredential" + ? { + companyId: requestedCompanyId, + projectId: originalRequest.params.projectId, + workspaceId: originalRequest.params.operation.source.id, + } : hostMethod === "state.get" ? { // Company-scoped state key — the shape a proactive gateway loop uses @@ -57,7 +63,13 @@ function sendNestedHostRequest(originalRequest, invocationId) { nestedRequest.paperclipInvocationId = "unknown-invocation"; } - pendingNested.set(nestedId, originalRequest.id); + pendingNested.set(nestedId, { + id: originalRequest.id, + ...(hostMethod === "projects.resolveGitCredential" ? { identity: { + runId: originalRequest.params.runId, + agentId: originalRequest.params.agentId, + } } : {}), + }); send(nestedRequest); } @@ -71,12 +83,12 @@ rl.on("line", (line) => { const message = JSON.parse(line); if (message.id && pendingNested.has(message.id)) { - const originalId = pendingNested.get(message.id); + const original = pendingNested.get(message.id); pendingNested.delete(message.id); if (message.error) { send({ jsonrpc: "2.0", - id: originalId, + id: original.id, error: message.error, }); return; @@ -84,8 +96,8 @@ rl.on("line", (line) => { send({ jsonrpc: "2.0", - id: originalId, - result: message.result, + id: original.id, + result: original.identity ? { identity: original.identity, credential: message.result } : message.result, }); return; } @@ -98,13 +110,13 @@ rl.on("line", (line) => { id: message.id, result: { ok: true, - supportedMethods: ["getData", "performAction", "resourceLifecycle", "workspaceOperation", "executeTool"], + supportedMethods: ["getData", "performAction", "resourceLifecycle", "workspaceOperation", "workspacePrepare", "executeTool"], }, }); return; } - if (["getData", "performAction", "resourceLifecycle", "workspaceOperation", "executeTool"].includes(method)) { + if (["getData", "performAction", "resourceLifecycle", "workspaceOperation", "workspacePrepare", "executeTool"].includes(method)) { sendNestedHostRequest(message, message.paperclipInvocation?.id); return; } diff --git a/server/src/__tests__/plugin-worker-manager.test.ts b/server/src/__tests__/plugin-worker-manager.test.ts index f635ca129fd..5fa5b3b8a5e 100644 --- a/server/src/__tests__/plugin-worker-manager.test.ts +++ b/server/src/__tests__/plugin-worker-manager.test.ts @@ -1,3 +1,4 @@ +import { withPluginWorkspaceReplay } from "../services/plugin-workspace-replay-context.js"; import { currentPluginInteractionOrigin } from "../services/plugin-interaction-context.js"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -336,6 +337,40 @@ describe("plugin-worker-manager stderr failure context", () => { } finally { await handle.stop().catch(() => undefined); } }); + it("keeps the provider request identity while scoping reservation replay Git access to project consent", async () => { + const projectGit = vi.fn(async (_params: unknown, context: { invocationScope?: { projectGit?: { runId?: string } } }) => { + if (context.invocationScope?.projectGit?.runId) throw new Error("Active task run is unavailable"); + return "project-consent"; + }); + const handle = createPluginWorkerHandle("test.plugin", { + entrypointPath: INVOCATION_SCOPE_WORKER_ENTRYPOINT, manifest: TEST_MANIFEST, config: {}, + instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, apiVersion: 1, + hostHandlers: { "projects.resolveGitCredential": projectGit as never }, + }); + const scope = { pluginId: "test.plugin", companyId: "company-a", projectId: "project-a", + projectWorkspaceId: "source-a", executionWorkspaceId: "workspace-a" }; + const params = { companyId: scope.companyId, projectId: scope.projectId, executionWorkspaceId: scope.executionWorkspaceId, + params: { mode: "echo", requestedCompanyId: scope.companyId, hostMethod: "projects.resolveGitCredential" }, + runId: "stopped-run", agentId: "agent-a", operation: { kind: "prepare", source: { id: scope.projectWorkspaceId } } }; + try { + await handle.start(); + for (const method of ["workspacePrepare", "workspaceOperation"] as const) { + const call = () => handle.call(method, params as never); + await expect(call()).rejects.toThrow("Active task run is unavailable"); + await expect(withPluginWorkspaceReplay({ ...scope, projectId: "another-project" }, call)) + .rejects.toThrow("Active task run is unavailable"); + await expect(withPluginWorkspaceReplay(scope, call)).resolves.toEqual({ + identity: { runId: "stopped-run", agentId: "agent-a" }, credential: "project-consent", + }); + expect(projectGit).toHaveBeenLastCalledWith( + { companyId: scope.companyId, projectId: scope.projectId, workspaceId: scope.projectWorkspaceId }, + expect.objectContaining({ invocationScope: { companyId: scope.companyId, + projectGit: { projectId: scope.projectId, workspaceId: scope.projectWorkspaceId, operation: "prepare" } } }), + ); + } + } finally { await handle.stop().catch(() => undefined); } + }); + it("passes performAction invocation scope to nested worker host calls", async () => { const companiesGet = vi.fn(async ( params: { companyId: string }, diff --git a/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts b/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts index 4d879705541..3961b5667a2 100644 --- a/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts +++ b/server/src/__tests__/plugin-workspace-terminal-retry.integration.test.ts @@ -29,10 +29,10 @@ function providerRefusal() { async function setup() { const f = await createPluginWorkspaceFixture(db); - const failures = { inspect: false, remove: false, unboundRemove: false, reopenFirst: false }; + const failures = { prepare: false, inspect: false, remove: false, unboundRemove: false, reopenFirst: false }; const workers = { ...f.workers, call: async (...args: Parameters) => { const params = args[2] as { agentId: string | null; operation: { kind: string; binding?: unknown } }; - if (params.operation.kind === "prepare" && params.agentId === null) { + if (params.operation.kind === "prepare" && failures.prepare) { f.calls.push(params); throw providerRefusal(); } @@ -123,6 +123,7 @@ function providerRefusal() { await expect(preparePluginWorkspace(db, s.workers, s.f.input)).rejects.toThrow("incomplete"); const [task] = await db.select().from(issues).where(eq(issues.id, s.f.issueId)); await s.finishTask(); + s.failures.prepare = true; const clock = Date.now(); await executionWorkspaceService(db, { pluginWorkerManager: s.workers, workspaceReaperCooldownDays: 0, now: () => new Date(clock) }) .sweepTerminalWorkspaces(100); @@ -139,6 +140,7 @@ function providerRefusal() { const s = await setup(); const reservation = await s.abandonReservation(); await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, s.f.runId)); + s.failures.prepare = true; await expect(inspectPluginWorkspace(db, s.workers, reservation)).rejects.toThrow(); expect(s.callsFor(reservation.id, "remove")).toHaveLength(0); expect(s.callsFor(reservation.id, "prepare")).toHaveLength(2); diff --git a/server/src/__tests__/plugin-workspaces.integration.test.ts b/server/src/__tests__/plugin-workspaces.integration.test.ts index dd05afef90a..7c21470b53d 100644 --- a/server/src/__tests__/plugin-workspaces.integration.test.ts +++ b/server/src/__tests__/plugin-workspaces.integration.test.ts @@ -1,3 +1,4 @@ +import { isPluginWorkspaceReplay } from "../services/plugin-workspace-replay-context.js"; import { createPluginWorkspaceFixture } from "./helpers/plugin-workspaces.js"; import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; @@ -6,6 +7,7 @@ import { createDb, subscribeDatabaseWork, environmentLeases, + environments, executionWorkspaces, heartbeatRuns, issues, @@ -640,8 +642,8 @@ const support = await getEmbeddedPostgresTestSupport(); inspectPluginWorkspace(db, f.workers, reservation), ).resolves.toMatchObject({ remotelyRecoverable: true }); expect(f.calls[1]).toMatchObject({ - runId: null, - agentId: null, + runId: f.runId, + agentId: f.input.agentId, operation: { kind: "prepare" }, }); expect(f.calls[1].operationId).toBe(f.calls[0].operationId); @@ -651,6 +653,73 @@ const support = await getEmbeddedPostgresTestSupport(); .where(eq(executionWorkspaces.id, reservation.id)); expect(recorded.providerRef).not.toBeNull(); }); + it.each(["inspect", "retry", "replacement"] as const)("recovers a lost Launcher prepare through %s", async (recovery) => { + const f = await fixture(); + const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId)); + await db.update(plugins).set({ manifestJson: { ...plugin.manifestJson, + capabilities: [...plugin.manifestJson.capabilities, "environment.drivers.register"], + environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", configSchema: { type: "object" }, + supportsTasks: true, sessionHandoffVersion: 1 }], + } }).where(eq(plugins.id, f.pluginId)); + const [environment] = await db.insert(environments).values({ name: `Workspace Launcher ${f.pluginId}`, driver: "plugin", + status: "active", config: { pluginKey: plugin.pluginKey, driverKey: "tasks" } }).returning(); + const taskLease = { environmentId: environment.id, driverKey: "tasks", providerLeaseId: randomUUID() }; + const leaseId = randomUUID(); + const input = { ...f.input, environment: { ...environment, driver: "plugin" as const }, + acquireTaskLease: async () => ({ leaseId, taskLease }) }; + let deleted = false; + const replacementTaskLease = { ...taskLease, providerLeaseId: randomUUID() }; + const replacementWorkers = { ...f.workers, call: vi.fn(async (...args) => { + const request = args[2] as (typeof f.calls)[number]; + if (deleted && request.taskLease?.providerLeaseId === taskLease.providerLeaseId) + throw new Error("Previous Launcher allocation is gone"); + const result = await f.workers.call(...args) as { kind: string }; + if (!deleted) throw new Error("Lost committed preparation result"); + return { ...result, created: false } as never; + }) }; + if (recovery !== "replacement") f.lose(); + await expect(preparePluginWorkspace(db, recovery === "replacement" ? replacementWorkers : f.workers, input)).rejects.toThrow("incomplete"); + const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const [reservation] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, task.executionWorkspaceId!)); + expect(reservation.providerRef).toBeNull(); + expect(reservation.metadata?.workspaceProviderPreparationPlacement).toMatchObject({ + generation: 0, runId: f.runId, agentId: f.input.agentId, taskLease, + }); + if (recovery === "inspect") { + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId)); + const workers = { ...f.workers, call: vi.fn(async (...args) => { + const request = args[2] as (typeof f.calls)[number]; + expect(isPluginWorkspaceReplay({ + pluginId: f.pluginId, companyId: f.companyId, projectId: f.input.projectId, + projectWorkspaceId: f.input.projectWorkspaceId, executionWorkspaceId: reservation.id, + })).toBe(request.operation.kind === "prepare"); + return f.workers.call(...args); + }) }; + await expect(inspectPluginWorkspace(db, workers, reservation)).resolves.toMatchObject({ remotelyRecoverable: true }); + } else if (recovery === "retry") { + await expect(preparePluginWorkspace(db, f.workers, { ...input, existingWorkspaceId: reservation.id })).resolves.toMatchObject({ id: reservation.id }); + } else { + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.runId)); + const [nextRun] = await db.insert(heartbeatRuns).values({ + companyId: f.companyId, issueId: f.issueId, agentId: f.input.agentId, status: "running", + }).returning(); + deleted = true; + await expect(preparePluginWorkspace(db, replacementWorkers, { + ...input, runId: nextRun.id, existingWorkspaceId: reservation.id, + acquireTaskLease: async () => ({ leaseId: randomUUID(), taskLease: replacementTaskLease }), + })).resolves.toMatchObject({ id: reservation.id }); + expect(replacementWorkers.call).toHaveBeenCalledTimes(2); + expect(f.calls[1]).toMatchObject({ runId: nextRun.id, agentId: f.input.agentId, taskLease: replacementTaskLease }); + expect(f.calls[1].operationId).not.toBe(f.calls[0].operationId); + } + if (recovery !== "replacement") expect(f.calls[1]).toMatchObject({ + runId: f.runId, agentId: f.input.agentId, taskLease, operationId: f.calls[0].operationId, + }); + expect(f.calls[1].operation).toEqual(f.calls[0].operation); + const [prepared] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, reservation.id)); + expect(prepared.providerRef).not.toBeNull(); + expect(prepared.metadata?.workspaceProviderPreparationPlacement).toBeUndefined(); + }); it("reconciles an unfinished reservation before applying PR-only cleanup eligibility", async () => { const f = await fixture(); const [plugin] = await db.select().from(plugins).where(eq(plugins.id, f.pluginId)); @@ -715,13 +784,23 @@ const support = await getEmbeddedPostgresTestSupport(); await expect( svc.removeWorkspace(f.input.projectId, f.input.projectWorkspaceId), ).rejects.toThrow("repository source"); + const placement = { generation: 0, runId: f.runId, agentId: f.input.agentId, + taskLease: { environmentId: randomUUID(), driverKey: "tasks", providerLeaseId: randomUUID() } }; + await db.update(executionWorkspaces).set({ + metadata: { ...workspace!.metadata, workspaceProviderPreparationPlacement: placement }, + }).where(eq(executionWorkspaces.id, workspace!.id)); const execution = executionWorkspaceService(db); const updated = await execution.update(workspace!.id, { metadata: { note: "caller edit", workspaceProvider: { workspaceRef: "foreign" }, + workspaceProviderPreparationPlacement: { ...placement, runId: randomUUID() }, }, }); + expect(updated!.metadata?.workspaceProviderPreparationPlacement).toBeUndefined(); + const [saved] = await db.select({ metadata: executionWorkspaces.metadata }) + .from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace!.id)); + expect(saved.metadata?.workspaceProviderPreparationPlacement).toEqual(placement); expect(updated!.metadata?.workspaceProvider).toEqual( workspace!.metadata?.workspaceProvider, ); diff --git a/server/src/__tests__/project-git-authorization.test.ts b/server/src/__tests__/project-git-authorization.test.ts index e250d3ae44c..d74f1fb5f76 100644 --- a/server/src/__tests__/project-git-authorization.test.ts +++ b/server/src/__tests__/project-git-authorization.test.ts @@ -305,6 +305,11 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu } await expect(f.service.resolve({ ...f.request, workspaceId: randomUUID() }, f.scope)).rejects.toThrow(); await expect(f.service.resolve({ ...f.request, companyId: randomUUID() }, f.scope)).rejects.toThrow(); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.input.runId)); + const preparation = { ...f.scope, workspaceId: f.workspace.id, operation: "prepare" as const }; + await expect(f.service.resolve(f.request, { ...preparation, runId: f.input.runId, agentId: f.input.agentId })) + .rejects.toThrow("Active task run is unavailable"); + await expect(f.service.resolve(f.request, preparation)).resolves.toBe("test-token-A"); }); it.each(["configured", "bound"])("authorizes a %s workspace selected from another project, but not an unselected source", async selection => { const f = await projectAuthorizationFixture(); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index da8d141adc1..77abc489223 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -50,6 +50,7 @@ import { type ProviderResourceDisposition, } from "./environment-runtime.js"; import { ENVIRONMENT_DRIVER_TRAITS } from "./environment-driver-traits.js"; +import type { PluginTaskDescription } from "./environment-task-runtime.js"; import { unlaunchedPortableTaskLease } from "./plugin-task-ownership.js"; import { resolveEnvironmentExecutionTarget, @@ -122,6 +123,7 @@ export interface EnvironmentAcquisitionResult { lease: EnvironmentLease; leaseContext: ReturnType; executionTransport: Record | null; + pluginTaskDescription?: PluginTaskDescription; } export interface EnvironmentRealizationResult { @@ -470,6 +472,32 @@ export function environmentRunOrchestrator( }; } + async function confirmWorkspaceTaskPlacement(acquisition: EnvironmentAcquisitionResult, input: { + companyId: string; + environmentId: string; + workspaceId: string; + signal?: AbortSignal; + }) { + const { environment, lease } = acquisition; + const driverKey = environment.config?.driverKey; + if (environment.id !== input.environmentId || typeof driverKey !== "string" || !lease.providerLeaseId) { + throw new Error("Workspace Launcher placement changed"); + } + const description = await environmentRuntime.task({ + companyId: input.companyId, leaseId: lease.id, operation: { kind: "describe" }, ownershipSignal: input.signal, + }); + const current = await environmentsSvc.getLeaseById(lease.id); + if (description.kind !== "description" || !current || current.providerLeaseId !== description.taskId || + current.status !== "active" || current.environmentId !== environment.id || + current.executionWorkspaceId !== input.workspaceId) { + throw new Error("Workspace Launcher lease is no longer active"); + } + return { + acquisition: { ...acquisition, lease: current, pluginTaskDescription: description }, + taskLease: { driverKey, environmentId: environment.id, providerLeaseId: description.taskId }, + }; + } + /** * Realize workspace in the environment and resolve the execution target. * @@ -880,6 +908,7 @@ export function environmentRunOrchestrator( acquireLease, resolveTransport, acquireForRun, + confirmWorkspaceTaskPlacement, realizeForRun, releaseForRun, diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4b72f1a78e3..0f0accc81e8 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -294,7 +294,7 @@ function preserveWorkspaceMetadataForPatch( } if (patch.metadata !== undefined) { nextPatch.metadata = { ...nextPatch.metadata }; - for (const key of ["workspaceProvider", "workspaceProviderOwner", "workspaceProviderSource", "workspaceProviderDestinationRef", "workspaceProviderBaseSha", "workspaceProviderRemoved", "workspaceProviderBranchMode", "workspaceProviderRemoval", "workspaceProviderRetired", "workspaceProviderRestoration", "lifecycleGeneration", "reopenPendingConsumption", "reopenPendingConsumptionSince"] as const) { + for (const key of ["workspaceProvider", "workspaceProviderOwner", "workspaceProviderSource", "workspaceProviderPreparationPlacement", "workspaceProviderDestinationRef", "workspaceProviderBaseSha", "workspaceProviderRemoved", "workspaceProviderBranchMode", "workspaceProviderRemoval", "workspaceProviderRetired", "workspaceProviderRestoration", "lifecycleGeneration", "reopenPendingConsumption", "reopenPendingConsumptionSince"] as const) { if (current.metadata?.[key] !== undefined) nextPatch.metadata[key] = current.metadata[key]; else delete nextPatch.metadata[key]; } @@ -1241,6 +1241,11 @@ function toExecutionWorkspace( runtimeServices: WorkspaceRuntimeService[] = [], deliveryState: ExecutionWorkspaceDeliveryState = "unknown", ): ExecutionWorkspace { + let metadata = row.metadata ?? null; + if (metadata?.workspaceProviderPreparationPlacement !== undefined) { + metadata = { ...metadata }; + delete metadata.workspaceProviderPreparationPlacement; + } return { id: row.id, companyId: row.companyId, @@ -1265,7 +1270,7 @@ function toExecutionWorkspace( cleanupEligibleAt: row.cleanupEligibleAt ?? null, cleanupReason: row.cleanupReason ?? null, config: readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null), - metadata: (row.metadata as Record | null) ?? null, + metadata, runtimeServices, createdAt: row.createdAt, updatedAt: row.updatedAt, diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index d970aa4e2bb..9c9b821a707 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -1,3 +1,4 @@ +import { armCrashPointForTest } from "@paperclipai/shared/crash-points"; import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js"; import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js"; @@ -7,13 +8,15 @@ import { randomUUID } from "node:crypto"; import { mkdtemp, rm, readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { agents, budgetPolicies, companies, completionContracts, nativeRunResults, costEvents, agentWakeupRequests, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; +import { agents, budgetPolicies, companies, completionContracts, nativeRunResults, costEvents, agentWakeupRequests, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, plugins, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js"; import { recoveryPostgresSupport as support, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { createPluginWorkspaceFixture } from "../../__tests__/helpers/plugin-workspaces.js"; +import type { PluginWorkspaceParams } from "@paperclipai/plugin-sdk"; import * as pluginWorkspaces from "../plugin-workspaces.js"; import * as legacyRecovery from "../legacy-execution-recovery.js"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; @@ -24,6 +27,7 @@ import { instanceSettingsService } from "../instance-settings.js"; import { executePaperclipNativeSession, NativeCancellationPendingRecoveryError, NativeControllerDetachedForRestartError, nativeSessionFailureSourceCode, nativeSessionFailureDisposition } from "../native-runtime/native-session-executor.js"; import { NativeRunnerOwnershipUnverifiedError } from "../native-runtime/native-runner-ownership.js"; import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError } from "../native-runtime/native-workspace-finalization-ownership.js"; +import { resumeNativeWorkspaceFinalization } from "../native-runtime/native-workspace-finalizer.js"; import { resolveHeartbeatNativeRuntimeMode } from "../native-runtime/runtime-mode.js"; import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; @@ -554,6 +558,7 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = name: "Plugin worktree", providerType: "plugin", providerRef: "provider:opaque-worktree", cwd: "/workspaces/plugin-only", mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch" }).returning(); input.workspace.persistedExecutionWorkspace = workspace; + const plan = vi.spyOn(pluginWorkspaces, "resolvePluginWorkspaceFinalization").mockResolvedValue({ kind: "inspect" }); const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockResolvedValue({ kind: "inspection", branchName: branch === "expected" ? "task-branch" : "other-branch", headSha: "a".repeat(40), hasDirtyTrackedFiles: true, hasUntrackedFiles: false, hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false }); @@ -566,8 +571,103 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = await expect(result).rejects.toThrow("plugin_workspace_branch_mismatch"); expect((await finalizations()).some(row => row.status === "succeeded")).toBe(false); } - expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef }), { agentId: input.agent.id, runId: input.run.id }); - } finally { inspect.mockRestore(); } + expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id, providerRef: workspace.providerRef }), + expect.objectContaining({ agentId: input.agent.id, runId: input.run.id })); + } finally { inspect.mockRestore(); plan.mockRestore(); } + }); + + it.each(["lost reply", "worker unavailable", "crash before dispatch", "ambiguous placement"])("replays inline plugin finalization after %s", async failure => { + const { input, run } = await fixture(true); + const issue = input.task.issueRef!; + const storage = await createPluginWorkspaceFixture(db, run.companyId); + const [plugin] = await db.select().from(plugins).where(eq(plugins.id, storage.pluginId)); + await db.update(plugins).set({ manifestJson: { + ...plugin.manifestJson, + workspaceProvider: { ...plugin.manifestJson.workspaceProvider!, finalizationVersion: 1 }, + environmentDrivers: [{ driverKey: "portable", displayName: "Portable", configSchema: { type: "object" }, supportsTasks: true, sessionHandoffVersion: 1 }], + } }).where(eq(plugins.id, storage.pluginId)); + const workspaceId = randomUUID(); + const binding = { pluginId: storage.pluginId, projectId: storage.input.projectId, executionWorkspaceId: workspaceId, + projectWorkspaceId: storage.input.projectWorkspaceId, sourceRevision: "revision", workspaceRef: "provider:portable" }; + const [workspace] = await db.insert(executionWorkspaces).values({ + id: workspaceId, companyId: run.companyId, projectId: binding.projectId, projectWorkspaceId: binding.projectWorkspaceId, + sourceIssueId: issue.id, name: "Portable worktree", providerType: "plugin", providerRef: binding.workspaceRef, + mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch", metadata: { workspaceProvider: binding }, + }).returning(); + input.workspace.persistedExecutionWorkspace = workspace; + await input.workspace.workspaceOperationRecorder.attachExecutionWorkspaceId(workspace.id); + await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id)); + const [boundRun] = await db.update(heartbeatRuns).set({ nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, run.id)).returning(); + input.run = boundRun; + await prepareNativeHeartbeatRun({ db, run: boundRun, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issue.id)); + const [accepted] = await db.insert(nativeRunResults).values({ + companyId: run.companyId, issueId: issue.id, runId: run.id, completionContractId: contract.id, + serverFingerprint: "inline-finalization", canonicalSha256: "inline-finalization", schemaStatus: "accepted", resultJson: {}, + }).returning(); + await db.insert(nativeRunFinalizations).values({ + companyId: run.companyId, issueId: issue.id, runId: run.id, phase: "result_accepted", resultId: accepted.id, + }); + const [environment] = await db.insert(environments).values({ name: `Portable Launcher ${failure}`, driver: "plugin", config: {} }).returning(); + const taskLease = { environmentId: environment.id, driverKey: "portable", providerLeaseId: "task-1" }; + const [lease] = await db.insert(environmentLeases).values({ + companyId: run.companyId, issueId: issue.id, heartbeatRunId: run.id, executionWorkspaceId: workspace.id, + environmentId: environment.id, providerLeaseId: taskLease.providerLeaseId, + metadata: { pluginId: storage.pluginId, agentId: run.agentId, driverKey: taskLease.driverKey, runnerTaskLease: true }, + }).returning(); + const duplicate = failure === "ambiguous placement" + ? (await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), providerLeaseId: "other-task" }).returning())[0] + : undefined; + let retry = false; + const dispatch = vi.fn(async (_pluginId: string, method: string, params: PluginWorkspaceParams) => { + expect(method).toBe("workspaceOperation"); + expect(params).toMatchObject({ runId: run.id, agentId: run.agentId, taskLease, operation: { kind: "finalize" } }); + if (!retry) { + expect(await finalizations()).toMatchObject([{ status: "running", metadata: { pluginWorkspaceTaskLease: taskLease } }]); + throw new Error("fixture reply lost"); + } + return { kind: "finalized", branchName: "task-branch", headSha: "a".repeat(40) }; + }); + const workers = { + getWorker: () => retry || failure === "lost reply" ? { status: "running", supportedMethods: ["workspaceOperation"] } : undefined, + call: dispatch, + } as unknown as NonNullable; + input.options.pluginWorkerManager = workers; + + const disarm = failure === "crash before dispatch" + ? armCrashPointForTest("workspace.finalize.after_start_before_copyback") : undefined; + try { + await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ + name: "NativeWorkspaceFinalizeScheduledError", reasonCode: "workspace_sync_out_failed", + }); + } finally { disarm?.(); } + expect(input.effects.onNativeWorkspaceFinalizeScheduled).toHaveBeenCalledOnce(); + if (duplicate) { + expect(await finalizations()).toHaveLength(0); + await db.delete(environmentLeases).where(eq(environmentLeases.id, duplicate.id)); + } else { + expect(await finalizations()).toEqual(expect.arrayContaining([ + expect.objectContaining({ status: "failed", metadata: expect.objectContaining({ pluginWorkspaceTaskLease: taskLease }) }), + ])); + } + if (failure === "crash before dispatch") { + expect(await finalizations()).toEqual(expect.arrayContaining([ + expect.objectContaining({ status: "running", metadata: expect.objectContaining({ pluginWorkspaceTaskLease: taskLease }) }), + ])); + } + if (failure !== "lost reply") expect(dispatch).not.toHaveBeenCalled(); + if (failure === "worker unavailable" || duplicate) { + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0].status).toBe("active"); + } else { + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date() }).where(eq(environmentLeases.id, lease.id)); + } + retry = true; + await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, run.id)); + await expect(resumeNativeWorkspaceFinalization({ db, runId: run.id, pluginWorkerManager: workers })).resolves.toMatchObject({ + status: "succeeded", metadata: { pluginWorkspaceTaskLease: taskLease }, + }); + expect(dispatch).toHaveBeenCalledTimes(failure === "lost reply" ? 2 : 1); + if (failure === "lost reply") expect(dispatch.mock.calls[1][2].operationId).toBe(dispatch.mock.calls[0][2].operationId); }); it("retains the same-run retry signal without recording a failed workspace", async () => { diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index a8ef0e0bcdb..93a7dfdd239 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -106,6 +106,7 @@ import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js"; import type { ProviderResourceDisposition } from "../environment-runtime.js"; import type { NativeExecutionInput, NativeSessionBackend } from "../../vendor/paperclip-runner/index.js"; import type { PluginWorkerManager } from "../plugin-worker-manager.js"; +import { resolvePluginWorkspaceFinalization } from "../plugin-workspaces.js"; import type { RealizedExecutionWorkspace } from "../workspace-runtime.js"; import type { reserveWarmNativeInstructionDirectory } from "../native-runtime/index.js"; @@ -297,15 +298,15 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; let preserveRunGatewayTokens = false; let pluginWorkspaceFinalization: Record | null = null; - const inspectFinalizeWorkspaceBranch = async () => { - const workspaceRecord = persistedExecutionWorkspace?.id - ? await executionWorkspacesSvc.getStoredById( - agent.companyId, - persistedExecutionWorkspace.id, - ) - : persistedExecutionWorkspace; + let pluginWorkspaceFinalizationPlan: Awaited>; + let pluginWorkspaceFinalizationRecord: typeof persistedExecutionWorkspace; + let pluginWorkspacePlacementFailed = false; + const inspectFinalizeWorkspaceBranch = async ( + workspaceRecord: typeof persistedExecutionWorkspace, + plan: Awaited>, + ) => { pluginWorkspaceFinalization = workspaceRecord - ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id) + ? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id, { taskLease: null, plan }) : null; if (pluginWorkspaceFinalization || readRemoteWorkspaceAuthority(workspaceRecord?.metadata) || workspaceRecord?.strategyType !== "git_worktree") return null; @@ -332,9 +333,23 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe restore?: () => Promise, ) => { if (adapterFinalizeOutcome) return; + const workspaceRecord = pluginWorkspaceFinalizationRecord ?? + ((status === "succeeded" || persistedExecutionWorkspace?.providerType === "plugin") && persistedExecutionWorkspace?.id + ? await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id).catch(error => { + if (persistedExecutionWorkspace.providerType === "plugin") pluginWorkspacePlacementFailed = true; + throw error; + }) + : persistedExecutionWorkspace); + const plan = pluginWorkspaceFinalizationPlan ??= workspaceRecord + ? await resolvePluginWorkspaceFinalization(db, workspaceRecord, { runId: run.id, agentId: agent.id }) + .catch(error => { pluginWorkspacePlacementFailed = true; throw error; }) + : undefined; + if (plan) pluginWorkspaceFinalizationRecord = workspaceRecord; + const pluginWorkspaceTaskLease = plan?.kind === "finalize" ? plan.taskLease : undefined; await workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, metadata: { adapterType: agent.adapterType, executionTargetKind: executionTarget?.kind ?? "local", + ...(pluginWorkspaceTaskLease ? { pluginWorkspaceTaskLease } : {}), ...(restore ? { owningService: "native_workspace_finalizer" } : {}), ...metadata }, run: async () => { await restore?.(); @@ -342,7 +357,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe let finalizeBranchRepairMetadata: Record | null = null; if (status === "succeeded") { - const branchInspection = await inspectFinalizeWorkspaceBranch(); + const branchInspection = await inspectFinalizeWorkspaceBranch(workspaceRecord, plan); if (branchInspection) { let inspection = branchInspection.inspection; const initialManagedGitWorktreeBranch = @@ -1231,7 +1246,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe .where(eq(nativeRunFinalizations.runId, run.id)) .limit(1) .then((rows) => rows[0]?.resultId ?? null); - if (proposedResult && (nativeWorkspaceSync || remoteWorkspaceAuthority)) { + if (proposedResult && (nativeWorkspaceSync || remoteWorkspaceAuthority || pluginWorkspacePlacementFailed || + (pluginWorkspaceFinalizationPlan?.kind === "finalize" && pluginWorkspaceFinalizationPlan.taskLease))) { const workspaceFailureMessage = adapterErr instanceof Error ? adapterErr.message : ""; const unrecoverable = diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 9f80accad3f..06b3ae06a1e 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -8,7 +8,7 @@ import { promisify } from "node:util"; import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, workspaceOperations, type Db } from "@paperclipai/db"; -import { eq, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -16,6 +16,8 @@ import { heartbeatService } from "../heartbeat.js"; import { registerServerAdapter, getServerAdapter, type AdapterExecutionContext } from "../../adapters/index.js"; import { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; import { environmentService } from "../environments.js"; +import { buildEnvironmentLeaseContext } from "../environment-runtime.js"; +import { callWorkspaceProvider, finalizePluginWorkspace, resolvePluginWorkspaceFinalization } from "../plugin-workspaces.js"; import { executionWorkspaceService, toExecutionWorkspace } from "../execution-workspaces.js"; import { instanceSettingsService } from "../instance-settings.js"; import { workspaceOperationService } from "../workspace-operations.js"; @@ -41,6 +43,7 @@ import { createStorageService } from "../../storage/service.js"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { selectHeartbeatEnvironment } from "./environment-selection.js"; import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js"; +import { NativeExecutionContext } from "../native-runtime/native-execution-context.js"; import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js"; import { projectService } from "../projects.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WorkspaceBusyDeferral } from "./retries.js"; @@ -955,17 +958,22 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace }); it.each([ - { fromDefault: false, mode: "isolated_workspace" as const, fresh: false }, - { fromDefault: false, mode: "isolated_workspace" as const, fresh: true }, - { fromDefault: true, mode: "isolated_workspace" as const, fresh: false }, - { fromDefault: true, mode: "operator_branch" as const, fresh: false }, - ])("prepares managed storage before acquisition (fromDefault: $fromDefault, mode: $mode, fresh: $fresh) without touching a Core checkout", async ({ fromDefault, mode: defaultMode, fresh }) => { + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: false }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: true, portable: false }, + { fromDefault: true, mode: "isolated_workspace" as const, fresh: false, portable: false }, + { fromDefault: true, mode: "operator_branch" as const, fresh: false, portable: false }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replace: true }, + { fromDefault: false, mode: "isolated_workspace" as const, fresh: false, portable: true, replaceAfterPrepare: true }, + ])("prepares managed storage with the selected Launcher (fromDefault: $fromDefault, mode: $mode, fresh: $fresh, portable: $portable, replace: $replace, replaceAfterPrepare: $replaceAfterPrepare)", async ({ fromDefault, mode: defaultMode, fresh, portable, replace, replaceAfterPrepare }) => { const { input, source, issue } = await fixture(true); - const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}`; + if (portable) input.agent.adapterType = "paperclip_runner"; + const pluginKey = `test.managed-storage-${fromDefault}-${defaultMode}-${fresh}-${portable}${replace ? "-replacement" : replaceAfterPrepare ? "-late-replacement" : ""}`; const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: "test-storage", version: "1.0.0", apiVersion: 1, status: "ready", categories: ["workspace"], manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Storage", description: "Storage", author: "Test", categories: ["workspace"], - workspaceProvider: { freshWorkspaceVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], - environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" } }], + workspaceProvider: { freshWorkspaceVersion: 1, finalizationVersion: 1, modes: ["isolated_workspace", "operator_branch"], strategies: ["git_worktree"], ...(fromDefault ? { defaultForProjects: { mode: defaultMode, strategy: "git_worktree" as const } } : {}) }, capabilities: ["workspaces.manage", "events.subscribe", "environment.drivers.register"], + environmentDrivers: [{ driverKey: "tasks", displayName: "Tasks", supportsTasks: true, configSchema: { type: "object" }, + ...(portable ? { sessionHandoffVersion: 1 as const } : {}) }], resourceLifecycle: ["project"], entrypoints: { worker: "./worker.js" } } }).returning(); const [environment] = await db.insert(environments).values({ name: `Storage tasks ${pluginKey}`, driver: "plugin", status: "active", config: { pluginKey, driverKey: "tasks", driverConfig: {} } }).returning(); @@ -986,28 +994,155 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace await rm(source, { recursive: true }); if (fresh) input.task.freshWorkspace = { mode: "fresh_isolated_workspace", retryIntentId: input.run.id, sourceRunId: input.run.id, retainedSourceIds: [] }; const sequence: string[] = []; - input.services.workspaceWorkers = { getWorker: () => ({ supportedMethods: ["workspaceOperation"] }), call: async (_: string, __: string, params: any) => { - sequence.push("prepare"); + const providerLeaseId = randomUUID(); + const readyTaskId = replace ? randomUUID() : providerLeaseId; + const replacementTaskId = randomUUID(); + let taskLease = { environmentId: environment.id, driverKey: "tasks", providerLeaseId: readyTaskId }; + const prepareRequests: Array<{ operation: Parameters[2]["operation"]; operationId: string }> = []; + const finalizationIds: string[] = []; + let executionWorkspaceId: string | undefined; + input.services.workspaceWorkers = { getWorker: () => ({ supportedMethods: [ + "workspaceOperation", "environmentTask", ...(portable ? ["workspacePrepare"] : []), + ] }), call: async (_: string, method: string, params: any) => { + if (method === "environmentAcquireLease") { + expect(replace).toBe(true); + expect(params.executionWorkspaceId).toBe(executionWorkspaceId); + sequence.push("replace"); + return { providerLeaseId: readyTaskId }; + } + if (method === "environmentTask") { + if (replace && params.operation.kind !== "describe") { + expect(params.taskId).toBe(providerLeaseId); + if (params.operation.kind === "status") + return { kind: "status", taskId: providerLeaseId, phase: "completed", executionStopped: true }; + expect(params.operation.kind).toBe("complete"); + return { kind: "accepted", taskId: providerLeaseId }; + } + expect(params.taskId).toBe(readyTaskId); + expect(params.operation.kind).toBe("describe"); + sequence.push("placement"); + return { kind: "description", taskId: readyTaskId, runner: { version: "test", digest: `sha256:${"a".repeat(64)}` } }; + } + expect(params.taskLease).toEqual(portable ? taskLease : undefined); + if (params.operation.kind === "finalize") { + sequence.push("finalize"); + finalizationIds.push(params.operationId); + return { kind: "finalized", headSha: "a".repeat(40), branchName: "test-WSP-1" }; + } + if (portable) { + expect(method).toBe("workspacePrepare"); + expect(params.phase).toBe("submit"); + expect(params.executionWorkspaceId).toBe(executionWorkspaceId); + prepareRequests.push(params); + } + sequence.push("workspace prepare"); + if (replaceAfterPrepare && prepareRequests.length === 2) throw new Error("lost workspace preparation reply"); return { kind: "prepared", binding: { pluginId: plugin.id, projectId: issue.projectId, executionWorkspaceId: params.executionWorkspaceId, - projectWorkspaceId: issue.projectWorkspaceId, sourceRevision: params.operation.source.revision, workspaceRef: "provider:checkout" }, + projectWorkspaceId: issue.projectWorkspaceId, sourceRevision: params.operation.source.revision, + workspaceRef: replaceAfterPrepare && prepareRequests.length === 3 ? "provider:replacement-checkout" : "provider:checkout" }, branchName: params.operation.branchName, baseSha: "a".repeat(40), created: true, branchCreatedByRuntime: true }; } } as any; + input.services.envOrchestrator = environmentRunOrchestrator(db, { + pluginWorkerManager: input.services.workspaceWorkers as never, + }); const acquire = input.services.envOrchestrator.acquireForRun; vi.spyOn(input.services.envOrchestrator, "acquireForRun").mockImplementation(async args => { - sequence.push("acquire"); expect(args.persistedExecutionWorkspace).toMatchObject({ providerType: "plugin", cwd: null }); - // Acquisition is a transport stub: this boundary test stops at handoff. - return acquire({ ...args, selectedEnvironmentId: args.localEnvironmentId }); + sequence.push("acquire"); + if (!portable) { + expect(args.persistedExecutionWorkspace).toMatchObject({ providerType: "plugin", cwd: null }); + return acquire({ ...args, selectedEnvironmentId: args.localEnvironmentId }); + } + expect(args.persistedExecutionWorkspace).toMatchObject({ mode: defaultMode, id: expect.any(String) }); + executionWorkspaceId = args.persistedExecutionWorkspace!.id; + const [lease] = await db.insert(environmentLeases).values({ + companyId: args.companyId, environmentId: environment.id, heartbeatRunId: args.heartbeatRunId, issueId: args.issueId, + executionWorkspaceId: args.persistedExecutionWorkspace!.id, provider: `plugin:${pluginKey}:tasks`, providerLeaseId, + status: "active", metadata: { driver: "plugin", pluginId: plugin.id, driverKey: "tasks", agentId: args.agentId, + runnerTaskLease: true, ...(replace ? { runnerTaskReused: true } : {}) }, + }).returning(); + return { environment: input.environment.selectedEnvironmentForConfig!, lease: (await environmentService(db).getLeaseById(lease.id))!, + leaseContext: buildEnvironmentLeaseContext(args), executionTransport: null }; }); vi.spyOn(input.services.envOrchestrator, "realizeForRun").mockImplementation(async args => { expect(args.executionWorkspace.providerBinding?.workspaceRef).toBe("provider:checkout"); + if (portable) expect(args.lease.providerLeaseId).toBe(readyTaskId); + if (replaceAfterPrepare) { + const runnerTask = async (operation: unknown, options?: { expectedTaskId?: string }) => { + const kind = (operation as { kind: string }).kind; + if (kind === "describe") { + sequence.push("replace after prepare"); + await db.update(environmentLeases).set({ providerLeaseId: replacementTaskId }).where(eq(environmentLeases.id, args.lease.id)); + return { kind: "description", taskId: replacementTaskId }; + } + if (kind === "prepare" && options?.expectedTaskId === replacementTaskId) { + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, executionWorkspaceId!)); + const lease = await environmentService(db).getLeaseById(args.lease.id); + expect(workspace?.providerRef).toBe("provider:replacement-checkout"); + expect(lease?.metadata?.workspaceProvider).toEqual(workspace?.metadata?.workspaceProvider); + } + sequence.push(`runner ${kind}`); + }; + return { lease: args.lease, workspaceRealization: {}, remoteExecution: null, + persistedExecutionWorkspace: args.persistedExecutionWorkspace, executionTarget: { + kind: "remote", transport: "sandbox", environmentId: environment.id, remoteCwd: ".", leaseId: args.lease.id, runnerTask, + } }; + } throw new Error("reached execution handoff"); }); + const portableSequence = ["acquire", ...(replace ? ["replace"] : []), "placement", "workspace prepare"]; + if (replaceAfterPrepare) { + const { executionTarget } = await prepareHeartbeatWorkspace(db, input); + if (executionTarget?.kind !== "remote" || executionTarget.transport !== "sandbox" || !executionTarget.runnerTask) + throw new Error("Plugin Runner target missing"); + const call = executionTarget.runnerTask; + await call({ kind: "prepare" }, { expectedTaskId: readyTaskId }); + expect(sequence).toEqual([...portableSequence, "runner prepare"]); + await expect(call({ kind: "describe" })).resolves.toMatchObject({ taskId: replacementTaskId }); + taskLease = { ...taskLease, providerLeaseId: replacementTaskId }; + const expiredContext = new NativeExecutionContext({ binding: { companyId: input.agent.companyId, + agentId: input.agent.id, runId: input.run.id } } as ConstructorParameters[0], 0); + await expect(call({ kind: "prepare" }, { expectedTaskId: replacementTaskId, executionContext: expiredContext })) + .rejects.toThrow("native_execution_startup_expired"); + expiredContext.finish(); + expect(sequence).toEqual([...portableSequence, "runner prepare", "replace after prepare"]); + await expect(call({ kind: "prepare" }, { expectedTaskId: replacementTaskId })).rejects.toThrow(); + expect(sequence).toEqual([...portableSequence, "runner prepare", "replace after prepare", "workspace prepare"]); + await call({ kind: "prepare" }, { expectedTaskId: replacementTaskId }); + await call({ kind: "resume" }, { expectedTaskId: replacementTaskId }); + await expect(call({ kind: "prepare" }, { expectedTaskId: randomUUID() })).rejects.toThrow("Workspace Launcher placement changed"); + expect(sequence).toEqual([...portableSequence, "runner prepare", "replace after prepare", "workspace prepare", "workspace prepare", "runner prepare", "runner resume"]); + expect(prepareRequests[1]?.operationId).not.toBe(prepareRequests[0]?.operationId); + expect(prepareRequests[2]?.operationId).toBe(prepareRequests[1]?.operationId); + return; + } await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached execution handoff"); - expect(sequence).toEqual(["prepare", "acquire"]); + expect(sequence).toEqual(portable ? portableSequence : ["workspace prepare", "acquire"]); await expect(access(source)).rejects.toThrow(); const [task] = await db.select().from(issues).where(eq(issues.id, issue.id)); const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, task.executionWorkspaceId!)); expect(workspace).toMatchObject({ providerType: "plugin", cwd: null, mode: defaultMode, strategyType: "git_worktree", branchName: fresh ? null : fromDefault ? "WSP-1-prepare-this-workspace" : "test-WSP-1" }); + if (portable) { + const finalization = { runId: input.run.id, agentId: input.agent.id }; + const plan = await resolvePluginWorkspaceFinalization(db, workspace, finalization); + expect(plan).toEqual({ kind: "finalize", taskLease }); + await expect(finalizePluginWorkspace(db, input.services.workspaceWorkers, workspace, { ...finalization, plan })) + .resolves.toMatchObject({ kind: "finalized" }); + expect(sequence).toEqual([...portableSequence, "finalize"]); + await db.update(environmentLeases).set({ status: "released" }).where(and( + eq(environmentLeases.companyId, input.agent.companyId), eq(environmentLeases.heartbeatRunId, input.run.id))); + await expect(finalizePluginWorkspace(db, input.services.workspaceWorkers, workspace, { + ...finalization, plan, + })).resolves.toMatchObject({ kind: "finalized" }); + expect(finalizationIds).toEqual([finalizationIds[0], finalizationIds[0]]); + const request = { workspace, pluginId: plugin.id, operation: prepareRequests[0]!.operation, + runId: input.run.id, agentId: input.agent.id }; + await callWorkspaceProvider(db, input.services.workspaceWorkers, { ...request, taskLease }); + expect(prepareRequests[1]?.operationId).toBe(prepareRequests[0]?.operationId); + taskLease = { ...taskLease, providerLeaseId: randomUUID() }; + await callWorkspaceProvider(db, input.services.workspaceWorkers, { ...request, taskLease }); + expect(prepareRequests).toHaveLength(3); + expect(prepareRequests[2]?.operationId).not.toBe(prepareRequests[0]?.operationId); + } }); async function installDefault() { const [plugin] = await db.insert(plugins).values({ pluginKey: "test.default-storage", packageName: "test-storage", version: "1.0.0", status: "ready", diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 6b18919a9af..d437a8cc273 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -7,6 +7,7 @@ import { remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js"; import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js"; import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js"; +import { NativeExecutionContext } from "../native-runtime/native-execution-context.js"; import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js"; import { reserveFreshPluginTaskWorkspace, stageProviderFreshWorkspace } from "../provider-fresh-workspace.js"; import { freshWorkspaceSchema, type PluginFreshWorkspace } from "@paperclipai/plugin-sdk"; @@ -160,7 +161,7 @@ export interface HeartbeatWorkspacePreparationInput { }; services: { workspaceWorkers?: WorkspaceWorkers; - envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "realizeForRun">; + envOrchestrator: Pick, "resolveEnvironment" | "acquireForRun" | "confirmWorkspaceTaskPlacement" | "realizeForRun">; executionWorkspacesSvc: Pick, "create" | "update" | "getStoredById" | "bindTaskWorkspace" | "prepareTaskRepositoriesForAdmission" | "assertTaskRepositoryPublicationAllowed" | "prepareConfiguredRepositoryCopy" | "assertTaskRepositoryRecoveryBinding">; workspaceOperationsSvc: Pick, "createRecorder">; resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"]; @@ -556,6 +557,59 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace previousWorkspaceId: previousProviderWorkspaceId }); } } + const assertAllowedEnvironment = (environment: Environment) => { + const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { + driver: environment.driver, + provider: + typeof environment.config?.provider === "string" + ? environment.config.provider + : null, + }); + if (!allowlistDecision.allowed) { + logger.error( + { + runId: run.id, + issueId, + agentId: agent.id, + environmentId: environment.id, + deniedDriver: allowlistDecision.deniedDriver, + deniedProvider: allowlistDecision.deniedProvider, + }, + "Execution allowlist denied the resolved environment; failing run", + ); + throw new Error(allowlistDecision.reason); + } + }; + const environmentAcquisitionInput = ( + persistedExecutionWorkspace: Parameters[0]["persistedExecutionWorkspace"], + projectId: string | null, + additionalWorkspaces: { projectId?: string | null }[] | undefined, + ) => ({ + companyId: agent.companyId, + selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id, + localEnvironmentId: localEnvironment.id, + adapterType: agent.adapterType, + adapterConfig: parseObject(agent.adapterConfig), + admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, + managedAiCredentialIdentity: input.config.managedAiCredentialIdentity, + runnerTaskResumeSessionId: input.config.runnerTaskResumeSessionId, + runnerTaskProjectIds: [...new Set([projectId, ...resolvedWorkspace.runnerTaskProjectIds ?? [], + ...(additionalWorkspaces ?? []).map(workspace => workspace.projectId)] + .filter((id): id is string => Boolean(id)))], + recoverPluginTask: !portablePreparationLeaseId && + (restartRecovery?.kind === "resume_dead_runner" || restartRecovery?.kind === "bootstrap_incomplete"), + issueId: issueId ?? null, + heartbeatRunId: run.id, + agentId: agent.id, + persistedExecutionWorkspace, + executionWorkspaceSettings: environmentExecutionWorkspaceSettings, + }); + const pluginRecovery = restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.remote.kind === "plugin_task" + ? restartRecovery.remote : portablePreparationLeaseId + ? { kind: "plugin_task" as const, leaseId: portablePreparationLeaseId, preparation: true as const } : undefined; + let workspaceAcquisition: Awaited> | undefined; + let workspaceAcquireStartedAtMs: number | undefined; + let ensureTaskWorkspace: ((taskId: string, assertOwned: () => void) => Promise | undefined) | undefined; const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, { companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId, issueId, runId: run.id, agentId: agent.id, responsibleUserId, @@ -563,6 +617,26 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace ...(providerFresh ? { fresh: providerFresh, expectedBindingRevision: previousProviderBindingRevision } : {}), selectedProviderPluginId: projectExecutionWorkspacePolicy?.storageProviderPluginId ?? null, environment: selectedEnvironmentForConfig, + acquireTaskLease: async workspace => { + if (!selectedEnvironmentForConfig) throw new Error("Workspace Launcher placement is unavailable"); + assertAllowedEnvironment(selectedEnvironmentForConfig); + await controllerLease.assertOwned(); + workspaceAcquireStartedAtMs = Date.now(); + if (restartRecovery?.kind === "reattach_remote_runner" && restartRecovery.runId !== run.id) + throw remoteRecoveryLeaseMismatch("workspace_preparation", "run_id"); + const acquisition = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ + ...environmentAcquisitionInput(workspace, resolvedWorkspace.projectId, resolvedWorkspace.additionalWorkspaces), + ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : {}), + })); + await controllerLease.assertOwned(); + const placement = await envOrchestrator.confirmWorkspaceTaskPlacement(acquisition, { + companyId: agent.companyId, environmentId: selectedEnvironmentForConfig.id, workspaceId: workspace.id, signal: input.signal, + }); + await controllerLease.assertOwned(); + workspaceAcquisition = placement.acquisition; + return { leaseId: placement.acquisition.lease.id, taskLease: placement.taskLease }; + }, + onTaskPrepared: ensure => { ensureTaskWorkspace = ensure; }, mode: requestedExecutionWorkspaceMode, strategy: resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig), branchMode: typeof parseObject(mergedConfig.workspaceStrategy).existingBranch === "string" ? "existing" : "create", @@ -1301,7 +1375,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace executionTarget: null, workspaceRestoreSource: null, }; } - const environmentAcquireStartedAtMs = Date.now(); + const environmentAcquireStartedAtMs = workspaceAcquireStartedAtMs ?? Date.now(); let acquiredEnvironment: Awaited< ReturnType >; @@ -1309,8 +1383,6 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace await controllerLease.assertOwned(); const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner" ? runOptions.nativeRestartRecovery : null; - const pluginRecovery = remoteRecovery?.remote.kind === "plugin_task" ? remoteRecovery.remote : portablePreparationLeaseId - ? { kind: "plugin_task" as const, leaseId: portablePreparationLeaseId, preparation: true as const } : undefined; const sandboxRecovery = remoteRecovery && remoteRecovery.remote.kind !== "plugin_task" ? remoteRecovery.remote : null; const recoveryWorkspace = sandboxRecovery ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null; @@ -1341,25 +1413,10 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace return envOrchestrator.acquireForRun(input); } }; - acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => acquireRetainedOrNew({ - companyId: agent.companyId, - selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id, - localEnvironmentId: localEnvironment.id, - adapterType: agent.adapterType, - adapterConfig: parseObject(agent.adapterConfig), - admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode, - managedAiCredentialIdentity: input.config.managedAiCredentialIdentity, - runnerTaskResumeSessionId: input.config.runnerTaskResumeSessionId, - runnerTaskProjectIds:[...new Set([executionWorkspace.projectId,...resolvedWorkspace.runnerTaskProjectIds ?? [], - ...(executionWorkspace.additionalWorkspaces ?? []).map(workspace=>workspace.projectId)] - .filter((id):id is string=>Boolean(id)))], - recoverPluginTask: !portablePreparationLeaseId && - (restartRecovery?.kind === "resume_dead_runner" || restartRecovery?.kind === "bootstrap_incomplete"), - issueId: issueId ?? null, - heartbeatRunId: run.id, - agentId: agent.id, - persistedExecutionWorkspace, - executionWorkspaceSettings: environmentExecutionWorkspaceSettings, + if (workspaceAcquisition && workspaceAcquisition.lease.executionWorkspaceId !== persistedExecutionWorkspace?.id) + throw new Error("Workspace Launcher lease changed before task admission"); + acquiredEnvironment = workspaceAcquisition ?? await timeWorkspace("workspace_environment_acquire", () => acquireRetainedOrNew({ + ...environmentAcquisitionInput(persistedExecutionWorkspace, executionWorkspace.projectId, executionWorkspace.additionalWorkspaces), ...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : sandboxRecovery && (recoveryWorkspace || remoteRecoveryLocation) ? { reattachRemoteLease: { leaseId: (remoteRecoveryLocation ?? recoveryWorkspace)!.leaseId, providerLeaseId: sandboxRecovery.providerLeaseId, @@ -1386,34 +1443,12 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace throw error; } const selectedEnvironment = acquiredEnvironment.environment; - // Defense-in-depth: re-check the actually-acquired environment against the - // execution allowlist. Even if selection were bypassed, a denied (local/ssh/ - // non-k8s) environment FAILS the run here rather than executing untrusted. - const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, { - driver: selectedEnvironment.driver, - provider: - typeof selectedEnvironment.config?.provider === "string" - ? selectedEnvironment.config.provider - : null, - }); - if (!allowlistDecision.allowed) { - logger.error( - { - runId: run.id, - issueId, - agentId: agent.id, - environmentId: selectedEnvironment.id, - deniedDriver: allowlistDecision.deniedDriver, - deniedProvider: allowlistDecision.deniedProvider, - }, - "Execution allowlist denied the resolved environment; failing run", - ); - throw new Error(allowlistDecision.reason); - } + assertAllowedEnvironment(selectedEnvironment); let activeEnvironmentLease = { environment: acquiredEnvironment.environment, lease: acquiredEnvironment.lease, leaseContext: acquiredEnvironment.leaseContext, + pluginTaskDescription: acquiredEnvironment.pluginTaskDescription, }; const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder( { @@ -1488,6 +1523,32 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); const workspaceRealization = realizationResult.workspaceRealization; const executionTarget = realizationResult.executionTarget; + if (ensureTaskWorkspace && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && executionTarget.runnerTask) { + if (executionTarget.leaseId !== acquiredEnvironment.lease.id) throw new Error("Workspace Launcher lease changed before task admission"); + const ensure = ensureTaskWorkspace; + const runnerTask = executionTarget.runnerTask; + executionTarget.runnerTask = (operation, options) => { + const kind = parseObject(operation).kind; + if (kind === "prepare" || kind === "resume") { + if (!options?.expectedTaskId) throw new Error("Workspace Launcher task identity unavailable"); + input.signal?.throwIfAborted(); + const assertOwned = () => { + input.signal?.throwIfAborted(); + const context = options.executionContext; + if (context !== undefined) { + if (!(context instanceof NativeExecutionContext)) throw new Error("native_execution_context_unavailable"); + context.assertScope({ companyId: agent.companyId, agentId: agent.id, runId: run.id }); + } + }; + const preparation = ensure(options.expectedTaskId, assertOwned); + if (preparation) return preparation.then(() => { + assertOwned(); + return runnerTask(operation, options); + }); + } + return runnerTask(operation, options); + }; + } if (remoteAuthority && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { const repositories = await timeWorkspace("workspace_repository_admission", () => executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId, issueId, diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index 489113814a0..0b638f5522e 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -57,7 +57,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { issueService } from "../issues.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; -import { resumeNativeWorkspaceFinalization, recordNativeWorkspaceFinalizationFailure } from "./native-workspace-finalizer.js"; +import { attemptNativeWorkspaceFinalization, recordNativeWorkspaceFinalizationFailure } from "./native-workspace-finalizer.js"; import { dismissObsoleteNativePolicyReviews } from "./obsolete-policy-reviews.js"; import { cleanupNativeWorkspaceSync, @@ -654,23 +654,21 @@ async function reconcileNativeFinalizationsPass( if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) { throw new Error("native_reconciliation_workspace_resume_policy_missing"); } - const operation = await resumeNativeWorkspaceFinalization({ + const outcome = await attemptNativeWorkspaceFinalization({ db, runId: row.runId, environmentRuntime: options.environmentRuntime, pluginWorkerManager: options.pluginWorkerManager, }); // Busy is ownership, not another failed export or retry-budget debit. - if (!operation) continue; - const workspaceFinalizeStatus = - operation.status === "succeeded" ? "succeeded" : "failed"; - if (workspaceFinalizeStatus === "failed") { - const failure = await recordNativeWorkspaceFinalizationFailure(db, row.runId, operation); + if (outcome.kind !== "completed" && outcome.kind !== "failed") continue; + if (outcome.kind === "failed" || outcome.operation.status !== "succeeded") { + const failure = await recordNativeWorkspaceFinalizationFailure(db, row.runId, outcome.operation); results.push({ ...failure, reconciliationAction: "resume_workspace_operation" as const, - workspaceOperationId: operation.id, - workspaceFinalizeStatus, + ...(outcome.kind === "completed" ? { workspaceOperationId: outcome.operation.id } : {}), + workspaceFinalizeStatus: "failed" as const, reconciliationDecision: recoveryDecision, }); if (failure.phase === "terminal_failure") { @@ -685,6 +683,8 @@ async function reconcileNativeFinalizationsPass( } continue; } + const { operation } = outcome; + const workspaceFinalizeStatus = "succeeded" as const; if (options.skipNativeResultCommit && row.coordinatorPhase !== "committed") continue; try { const finalized = await finalizeNativeRun({ diff --git a/server/src/services/native-runtime/native-plugin-workspace-finalization.ts b/server/src/services/native-runtime/native-plugin-workspace-finalization.ts index 5eec1afccb0..c33a1fcf0ad 100644 --- a/server/src/services/native-runtime/native-plugin-workspace-finalization.ts +++ b/server/src/services/native-runtime/native-plugin-workspace-finalization.ts @@ -8,9 +8,10 @@ export async function inspectPluginWorkspaceForFinalization( workspace: typeof executionWorkspaces.$inferSelect, agentId: string, runId: string, + placement?: Pick[3], "taskLease" | "plan">, ): Promise | null> { if (workspace.providerType === "plugin") { - const inspection = await finalizePluginWorkspace(db, workers, workspace, { agentId, runId }); + const inspection = await finalizePluginWorkspace(db, workers, workspace, { agentId, runId, ...placement }); if (workspace.strategyType === "git_worktree" && (!workspace.branchName || inspection.branchName !== workspace.branchName)) { throw new Error("plugin_workspace_branch_mismatch"); diff --git a/server/src/services/native-runtime/native-workspace-finalization-work.ts b/server/src/services/native-runtime/native-workspace-finalization-work.ts index 6282357c65b..fd943e15eb0 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-work.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-work.ts @@ -92,7 +92,7 @@ export function nativeWorkspaceFinalizationWork(db: Db, options: { } return null; } - if (outcome.kind !== "completed" || outcome.operation.status === "succeeded") return null; + if (outcome.kind === "inactive" || (outcome.kind === "completed" && outcome.operation.status === "succeeded")) return null; const failure = await recordNativeWorkspaceFinalizationFailure(db, current.id, outcome.operation); if (failure.phase === "terminal_failure") { await options.onWorkspaceSettled({ runId: row.id, companyId: row.companyId, agentId: row.agentId, succeeded: false }); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index 6897b7637b6..5b5e18b3049 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -1,3 +1,4 @@ +import { armCrashPointForTest } from "@paperclipai/shared/crash-points"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import * as pluginWorkspaces from "../plugin-workspaces.js"; @@ -23,6 +24,8 @@ import { companies, completionContracts, createDb, + environmentLeases, + environments, executionWorkspaces, heartbeatRuns, issues, @@ -30,6 +33,7 @@ import { nativeRunResults, issueRecoveryActions, issueThreadInteractions, + plugins, projects, workspaceOperations, workAssessments, @@ -37,6 +41,7 @@ import { } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; +import { nativeWorkspaceFinalizationWork } from "./native-workspace-finalization-work.js"; import { reconcileNativeFinalizations } from "./native-finalization-reconciler.js"; describe("native workspace finalization recovery", () => { @@ -44,7 +49,6 @@ describe("native workspace finalization recovery", () => { let connectionString: string; let db: ReturnType; let workspaceRoot: string; - let priorLogRoot: string | undefined; const companyId = randomUUID(); const foreignCompanyId = randomUUID(); @@ -55,10 +59,11 @@ describe("native workspace finalization recovery", () => { async function seedRun(input: { executionWorkspaceId: string; title: string; + runId?: string; }) { const issueId = randomUUID(); const contractId = randomUUID(); - const runId = randomUUID(); + const runId = input.runId ?? randomUUID(); const resultId = randomUUID(); await db.insert(issues).values({ @@ -123,11 +128,7 @@ describe("native workspace finalization recovery", () => { workspaceRoot = await fs.mkdtemp( path.join(os.tmpdir(), "paperclip-native-workspace-finalizer-"), ); - priorLogRoot = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; - process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = path.join( - workspaceRoot, - "operation-logs", - ); + vi.stubEnv("WORKSPACE_OPERATION_LOG_BASE_PATH", path.join(workspaceRoot, "operation-logs")); const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-"); connectionString = external ?? temporary!.connectionString; @@ -168,11 +169,7 @@ describe("native workspace finalization recovery", () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); await fs.rm(workspaceRoot, { recursive: true, force: true }); - if (priorLogRoot === undefined) { - delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; - } else { - process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = priorLogRoot; - } + vi.unstubAllEnvs(); }); async function seedAbandonedCopyback() { @@ -752,14 +749,70 @@ describe("native workspace finalization recovery", () => { mode: "isolated_workspace", strategyType: providerType === "plugin" ? "git_worktree" : "project_primary", branchName: "task-branch", metadata: providerType === "plugin" ? {} : { runnerTaskWorkspaceOwner: { agentId } } }).returning(); const seeded = await seedRun({ executionWorkspaceId: workspace.id, title: "Plugin result before controller restart" }); + const plan = vi.spyOn(pluginWorkspaces, "resolvePluginWorkspaceFinalization").mockResolvedValue({ kind: "inspect" }); const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockResolvedValue({ kind: "inspection", branchName: "task-branch", headSha: "a".repeat(40), hasDirtyTrackedFiles: true, hasUntrackedFiles: false, hasLocalOnlyData: true, isMergedIntoBase: false, remotelyRecoverable: false }); try { expect(await resumeNativeWorkspaceFinalization({ db, runId: seeded.runId })).toMatchObject({ status: "succeeded" }); - if (providerType === "plugin") expect(inspect).toHaveBeenCalledWith(db, undefined, expect.objectContaining({ id: workspace.id }), { agentId, runId: seeded.runId }); + if (providerType === "plugin") expect(inspect).toHaveBeenCalledWith(db, undefined, + expect.objectContaining({ id: workspace.id }), expect.objectContaining({ agentId, runId: seeded.runId })); else expect(inspect).not.toHaveBeenCalled(); - } finally { inspect.mockRestore(); } + } finally { inspect.mockRestore(); plan.mockRestore(); } + }); + + it.each(["pre-placement request", "crash before dispatch"] as const)("replays the recorded route after %s", async scenario => { + const pluginId = randomUUID(), environmentId = randomUUID(), workspaceId = randomUUID(); + const binding = { pluginId, projectId, executionWorkspaceId: workspaceId, projectWorkspaceId: randomUUID(), + sourceRevision: "fixture-revision", workspaceRef: "provider:pre-placement" }; + await db.insert(plugins).values({ + id: pluginId, pluginKey: pluginId, packageName: "test-storage", version: "1.0.0", status: "ready", + manifestJson: { capabilities: ["workspaces.manage"], resourceLifecycle: ["project"], + workspaceProvider: { finalizationVersion: 1 }, + environmentDrivers: [{ driverKey: "portable", sessionHandoffVersion: 1 }] } as never, + }); + await db.insert(environments).values({ id: environmentId, name: `Portable tasks ${environmentId}`, driver: "plugin" }); + const [workspace] = await db.insert(executionWorkspaces).values({ + id: workspaceId, companyId, projectId, name: "Pre-placement worktree", providerType: "plugin", providerRef: binding.workspaceRef, + mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch", metadata: { workspaceProvider: binding }, + }).returning(); + const { runId, issueId } = await seedRun({ executionWorkspaceId: workspaceId, title: "Workspace finalization interrupted during upgrade" }); + await db.insert(environmentLeases).values({ + companyId, environmentId, executionWorkspaceId: workspaceId, heartbeatRunId: runId, issueId, + status: "active", provider: "plugin:fixture:portable", providerLeaseId: "live-task", + metadata: { pluginId, agentId, driverKey: "portable", runnerTaskLease: true }, + }); + const result = { kind: "finalized", branchName: "task-branch", headSha: "a".repeat(40) }; + const worker = vi.fn().mockResolvedValue(result); + const workers = { getWorker: () => ({ supportedMethods: ["workspaceOperation"] }), call: worker } as unknown as pluginWorkspaces.WorkspaceWorkers; + const resume = () => resumeNativeWorkspaceFinalization({ db, runId, pluginWorkerManager: workers }); + if (scenario === "pre-placement request") { + worker.mockRejectedValueOnce(new Error("fixture reply lost")); + await expect(pluginWorkspaces.callWorkspaceProvider(db, workers, { + workspace, runId, agentId, operation: { kind: "finalize", binding }, + })).rejects.toThrow(); + await db.insert(workspaceOperations).values({ + companyId, executionWorkspaceId: workspaceId, heartbeatRunId: runId, issueId, + phase: "workspace_finalize", status: "running", metadata: {}, + }); + await expect(resume()).resolves.toMatchObject({ status: "succeeded" }); + const requests = worker.mock.calls.map(([, , request]) => request); + expect(requests).toHaveLength(2); + expect(requests[1]?.operationId).toBe(requests[0]?.operationId); + expect(requests.every(request => request.taskLease === undefined)).toBe(true); + } else { + const disarm = armCrashPointForTest("workspace.finalize.after_start_before_copyback"); + try { await expect(resume()).rejects.toThrow("workspace.finalize.after_start_before_copyback"); } + finally { disarm(); } + expect(worker).not.toHaveBeenCalled(); + const taskLease = { environmentId, driverKey: "portable", providerLeaseId: "live-task" }; + const [interrupted] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, runId)); + expect(interrupted).toMatchObject({ status: "running", metadata: { pluginWorkspaceTaskLease: taskLease } }); + await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.heartbeatRunId, runId)); + await expect(resume()).resolves.toMatchObject({ status: "succeeded", metadata: { pluginWorkspaceTaskLease: taskLease } }); + expect(worker).toHaveBeenCalledExactlyOnceWith(pluginId, "workspaceOperation", + expect.objectContaining({ taskLease, runId, agentId, operation: { kind: "finalize", binding } }), 40_000); + } }); it.each(["synthetic plugin unavailable", "plugin_workspace_branch_mismatch", "plugin_workspace_owner_mismatch"])("defers plugin failure without aborting other result recovery: %s", async message => { @@ -771,6 +824,7 @@ describe("native workspace finalization recovery", () => { ]).returning(); const blocked = await seedRun({ executionWorkspaceId: unavailable.id, title: "Plugin is unavailable after restart" }); const ready = await seedRun({ executionWorkspaceId: healthy.id, title: "Other result can recover" }); + const plan = vi.spyOn(pluginWorkspaces, "resolvePluginWorkspaceFinalization").mockResolvedValue({ kind: "inspect" }); const inspect = vi.spyOn(pluginWorkspaces, "finalizePluginWorkspace").mockRejectedValue(new Error(message)); const code = message.startsWith("plugin_workspace_") ? message : "plugin_workspace_inspection_failed"; try { @@ -783,7 +837,43 @@ describe("native workspace finalization recovery", () => { .toMatchObject([{ status: "failed", stderrExcerpt: `${code}\n` }]); expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, ready.runId))) .toMatchObject([{ status: "succeeded" }]); - } finally { inspect.mockRestore(); } + } finally { inspect.mockRestore(); plan.mockRestore(); } + }); + + it("accounts for a removed workspace plugin without blocking other accepted runs", async () => { + const workspaceId = randomUUID(); + const binding = { pluginId: randomUUID(), projectId, executionWorkspaceId: workspaceId, projectWorkspaceId: randomUUID(), + sourceRevision: "fixture-revision", workspaceRef: "provider:removed-plugin" }; + const [unavailable, healthy] = await db.insert(executionWorkspaces).values([ + { id: workspaceId, companyId, projectId, name: "Removed plugin", providerType: "plugin", providerRef: binding.workspaceRef, + mode: "isolated_workspace", strategyType: "git_worktree", branchName: "task-branch", metadata: { workspaceProvider: binding } }, + { companyId, projectId, name: "Healthy task", providerType: "adapter_managed", cwd: "/workspaces/healthy-task", + mode: "shared_workspace", strategyType: "project_primary", metadata: { runnerTaskWorkspaceOwner: { agentId } } }, + ]).returning(); + const blocked = await seedRun({ executionWorkspaceId: unavailable.id, title: "Removed plugin", runId: `0${randomUUID().slice(1)}` }); + const ready = await seedRun({ executionWorkspaceId: healthy.id, title: "Healthy finalization", runId: `f${randomUUID().slice(1)}` }); + await expect(reconcileNativeFinalizations(db, [blocked.runId, ready.runId])).resolves.toBeDefined(); + expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, ready.runId))) + .toMatchObject([{ status: "succeeded" }]); + const coordinator = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, blocked.runId)) + .then(([row]) => row); + expect(await coordinator()).toMatchObject({ + phase: "retryable_failure", nextAttemptAt: expect.any(Date), + failureCode: "native_workspace_sync_out_failed", failureDetail: { workspaceFinalizeAttempt: 1 }, + }); + + const onWorkspaceSettled = vi.fn(async () => {}); + const work = nativeWorkspaceFinalizationWork(db, { onWorkspaceSettled }); + for (const attempt of [2, 3]) { + await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, blocked.runId)); + const [entry] = await work.load({ companyId, runId: blocked.runId }); + expect(await work.check(entry)).toEqual(attempt === 2 ? expect.objectContaining({ id: blocked.runId }) : null); + expect(await coordinator()).toMatchObject({ + phase: attempt === 2 ? "retryable_failure" : "terminal_failure", failureDetail: { workspaceFinalizeAttempt: attempt }, + }); + } + expect(onWorkspaceSettled).toHaveBeenCalledExactlyOnceWith({ runId: blocked.runId, companyId, agentId, succeeded: false }); + expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, blocked.runId))).toHaveLength(0); }); it("records directory-only recovery without inventing an execution-workspace foreign key", async () => { diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index 9bdd2c2aaf7..ef154b04c8a 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -3,7 +3,7 @@ import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-wor import { remoteWorkspaceFailureCause } from "../remote-workspace-transport.js"; import { recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js"; -import type { WorkspaceWorkers } from "../plugin-workspaces.js"; +import { resolvePluginWorkspaceFinalization, type WorkspaceWorkers } from "../plugin-workspaces.js"; import { restoreNativeWorkspaceBestEffort } from "./native-workspace-best-effort.js"; import { withNativeWorkspaceFinalizationOwnership, type NativeWorkspaceFinalizationOwnership } from "./native-workspace-finalization-ownership.js"; import { settleSupersededNativeWorkspaceFinalizations } from "./native-workspace-finalization-supersession.js"; @@ -11,6 +11,7 @@ import { logger } from "../../middleware/logger.js"; import fs from "node:fs/promises"; import { and, desc, eq } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; +import { pluginWorkspaceParamsSchema } from "@paperclipai/plugin-sdk"; import { executionWorkspaces, heartbeatRuns, @@ -157,6 +158,21 @@ export async function attemptNativeWorkspaceFinalization(input: { )).limit(1).then((rows) => rows[0] ?? null) : null; const cwd = workspace?.providerRef ?? workspace?.cwd ?? previous?.cwd ?? null; + let pluginFinalizationPlan: Awaited>; + if (workspace && !nativeWorkspaceSync && !readRemoteWorkspaceAuthority(workspace.metadata)) { + try { + const taskLease = previous + ? pluginWorkspaceParamsSchema.shape.taskLease.parse(previous.metadata?.pluginWorkspaceTaskLease) ?? null + : undefined; + pluginFinalizationPlan = await resolvePluginWorkspaceFinalization(input.db, workspace, { + runId: input.runId, agentId: bound.agentId, taskLease, + }); + } catch { + await ownership.assertHeld(); + return { kind: "failed", operation: { stderrExcerpt: "plugin_workspace_inspection_failed" } } as const; + } + } + const recordedTaskLease = pluginFinalizationPlan?.kind === "finalize" ? pluginFinalizationPlan.taskLease : undefined; const recorder = workspaceOperationService(input.db).createRecorder({ companyId: bound.companyId, @@ -173,6 +189,7 @@ export async function attemptNativeWorkspaceFinalization(input: { metadata: { resumedFromOperationId: previous?.id ?? null, owningService: "native_workspace_finalizer", + ...(recordedTaskLease ? { pluginWorkspaceTaskLease: recordedTaskLease } : {}), observation: workspace?.strategyType === "git_worktree" ? "managed_git_worktree_branch" : "workspace_directory", @@ -307,7 +324,10 @@ export async function attemptNativeWorkspaceFinalization(input: { let pluginWorkspace: Record | null; try { pluginWorkspace = workspace - ? await inspectPluginWorkspaceForFinalization(input.db, input.pluginWorkerManager, workspace, bound.agentId, input.runId) + ? await inspectPluginWorkspaceForFinalization(input.db, input.pluginWorkerManager, workspace, bound.agentId, input.runId, { + taskLease: null, + plan: pluginFinalizationPlan, + }) : null; } catch (error) { await ownership.assertHeld(); diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index d28b800d5be..aad28a6e8b4 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -1,3 +1,4 @@ +import { isPluginWorkspaceReplay } from "./plugin-workspace-replay-context.js"; import { withPluginInteractionOrigin } from "./plugin-interaction-context.js"; import { pluginIdleManifestGeneration, type IdlePluginCatalogEntry } from "./plugin-idle-catalog.js"; import { createHash, randomUUID } from "node:crypto"; @@ -1295,9 +1296,13 @@ export function createPluginWorkerHandle( const workspaceId = isRecord(source) ? readNonEmptyString(operation.kind === "prepare" ? source.id : source.projectWorkspaceId) : null; const projectId = readNonEmptyString(params.projectId); if (projectId && workspaceId && (operation.kind === "prepare" || operation.kind === "inspect" || operation.kind === "remove")) { - // Only foreground preparation may use a run. Maintenance uses project consent. - const runId = operation.kind === "prepare" ? readNonEmptyString(params.runId) : null; - const agentId = operation.kind === "prepare" ? readNonEmptyString(params.agentId) : null; + const executionWorkspaceId = readNonEmptyString(params.executionWorkspaceId); + // Reconciliation preserves the provider's request identity while Git uses project consent. + const foreground = operation.kind === "prepare" && !(executionWorkspaceId && isPluginWorkspaceReplay({ + pluginId, companyId: directCompanyId, projectId, projectWorkspaceId: workspaceId, executionWorkspaceId, + })); + const runId = foreground ? readNonEmptyString(params.runId) : null; + const agentId = foreground ? readNonEmptyString(params.agentId) : null; if (Boolean(runId) !== Boolean(agentId)) return { companyId: directCompanyId }; return { companyId: directCompanyId, projectGit: { projectId, workspaceId, operation: operation.kind, ...(runId && agentId ? { runId, agentId } : {}) } }; diff --git a/server/src/services/plugin-workspace-preparation.ts b/server/src/services/plugin-workspace-preparation.ts index 895fc6c0e4f..9d69c68bb15 100644 --- a/server/src/services/plugin-workspace-preparation.ts +++ b/server/src/services/plugin-workspace-preparation.ts @@ -1,14 +1,128 @@ +import { createHash, randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; -import { parseWorkspacePrepareResult, parseWorkspaceProviderResult, type PluginWorkspaceParams } from "@paperclipai/plugin-sdk"; -import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import { and, eq } from "drizzle-orm"; +import { executionWorkspaces, type Db } from "@paperclipai/db"; +import { readWorkspaceProviderBinding } from "@paperclipai/shared"; +import { + parseWorkspacePrepareResult, + parseWorkspaceProviderResult, + workspaceProviderOperationSchema, + type PluginWorkspaceOperation, + type PluginWorkspaceParams, + type PluginWorkspaceResult, +} from "@paperclipai/plugin-sdk"; +import { conflict } from "../errors.js"; +import { requireWorkspaceProviderWorker, type WorkspaceWorkers } from "./project-workspace-provider.js"; import { WorkspaceProviderFailureError, workspaceProviderCallFailure } from "./workspace-provider-failure.js"; +type Workspace = typeof executionWorkspaces.$inferSelect; type ProviderRequest = Omit; -type WorkspaceWorker = Pick; const legacyWorkspaceOperationMs = 300_000; const legacyRpcGraceMs = 10_000; -export async function invokeWorkspaceProvider(workers: WorkspaceWorker, params: ProviderRequest) { +/** No paths or provider credentials are accepted from task callers. */ +export async function callWorkspaceProvider( + db: Db, + workers: WorkspaceWorkers | undefined, + input: { + workspace: Workspace; + operation: PluginWorkspaceOperation; + pluginId?: string; + advanceGeneration?: boolean; + runId?: string | null; + agentId?: string | null; + removalOperationId?: string; + taskLease?: PluginWorkspaceParams["taskLease"]; + assertOwned?: () => void; + }, +) { + const { workspace } = input; + const binding = readWorkspaceProviderBinding(workspace.metadata); + const pluginId = binding?.pluginId ?? input.pluginId; + if (!pluginId || !workspace.projectId) + throw conflict("Workspace storage ownership is missing"); + const selectedProvider = await requireWorkspaceProviderWorker(db, workers, workspace.companyId, pluginId, true); + const verifyExisting = input.operation.kind === "prepare" && input.operation.verifyExisting === true; + if (verifyExisting && selectedProvider.manifestJson.workspaceProvider?.workspaceVerificationVersion !== 1) + throw conflict("Workspace verification is unavailable"); + if (input.operation.kind === "prepare" && input.operation.fresh && selectedProvider.manifestJson.workspaceProvider?.freshWorkspaceVersion !== 1) + throw conflict("Fresh workspace preparation is unavailable"); + if (input.operation.kind === "finalize" && + (selectedProvider.manifestJson.workspaceProvider?.finalizationVersion !== 1 || !input.runId || !input.agentId)) + throw conflict("Workspace finalization is unavailable"); + const generation = + typeof workspace.metadata?.lifecycleGeneration === "number" + ? workspace.metadata.lifecycleGeneration + : 0; + const operationGeneration = generation + (input.advanceGeneration ? 1 : 0); + const operation = workspaceProviderOperationSchema.parse(input.operation); + if (input.removalOperationId !== undefined && (operation.kind !== "remove" || + input.removalOperationId !== createHash("sha256").update(JSON.stringify([workspace.id, operationGeneration, operation])).digest("hex"))) + throw conflict("Workspace removal identity does not match its recorded payload"); + if ((operation.kind === "inspect" || operation.kind === "remove") && input.removalOperationId === undefined + && selectedProvider.manifestJson.workspaceProvider?.recoveryPullRequestVersion !== 1) delete operation.recoveryPullRequest; + if (operation.kind === "prepare" && operation.fresh && !verifyExisting) { + if (!workspace.sourceIssueId) throw conflict("Fresh workspace task ownership is missing"); + const { stageProviderFreshWorkspace } = await import("./provider-fresh-workspace.js"); + operation.fresh = await stageProviderFreshWorkspace({ db, companyId: workspace.companyId, + issueId: workspace.sourceIssueId, ...operation.fresh, existing: operation.fresh }); + } + const params: ProviderRequest = { + pluginId, + companyId: workspace.companyId, + projectId: workspace.projectId, + executionWorkspaceId: workspace.id, + operationId: input.removalOperationId ?? ( + operation.kind === "inspect" + ? randomUUID() + : createHash("sha256") + .update( + JSON.stringify([workspace.id, operationGeneration, operation, + ...(operation.kind === "finalize" ? [input.runId, input.agentId] : []), + ...(input.taskLease ? [input.taskLease.environmentId, input.taskLease.driverKey, input.taskLease.providerLeaseId] : [])]), + ) + .digest("hex")), + generation: operationGeneration, + runId: input.runId ?? null, + agentId: input.agentId ?? null, + ...(input.taskLease ? { taskLease: input.taskLease } : {}), + operation, + }; + input.assertOwned?.(); + let result: PluginWorkspaceResult; + try { + result = await invokeWorkspaceProvider(workers!, params); + } catch (error) { + // A worker that never received the call, a lost reply and a provider-reported + // failure need different recovery; only a closed provider reason crosses over. + if (error instanceof WorkspaceProviderFailureError) throw error; + throw workspaceProviderCallFailure(error); + } + // Recheck enablement and ownership after an external effect, including an ambiguous retry. + await requireWorkspaceProviderWorker(db, workers, workspace.companyId, pluginId); + const [fresh] = await db + .select() + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, workspace.id), + eq(executionWorkspaces.companyId, workspace.companyId), + ), + ); + if ( + !fresh || + fresh.status !== workspace.status || + (fresh.metadata?.lifecycleGeneration ?? 0) !== generation || + fresh.providerRef !== workspace.providerRef + ) { + throw conflict("Workspace changed during storage preparation"); + } + if (result.kind === "prepared" && result.binding.pluginId !== pluginId) + throw conflict("Workspace provider ownership changed"); + return result; +} + +export async function invokeWorkspaceProvider(workers: WorkspaceWorkers, params: ProviderRequest) { const verifyExisting = params.operation.kind === "prepare" && params.operation.verifyExisting; if (!verifyExisting && params.operation.kind === "prepare" && workers.getWorker(params.pluginId)?.supportedMethods?.includes("workspacePrepare")) { const result = await prepareWorkspaceWithPolling(workers, params); @@ -21,7 +135,7 @@ export async function invokeWorkspaceProvider(workers: WorkspaceWorker, params: )); } -async function prepareWorkspaceWithPolling(workers: Pick, params: ProviderRequest) { +async function prepareWorkspaceWithPolling(workers: Pick, params: ProviderRequest) { const rpcTimeout = 15_000; const submit = { ...params, phase: "submit" as const, deadline: new Date(Date.now() + rpcTimeout).toISOString() }; const receipt = parseWorkspacePrepareResult(submit, await workers.call(params.pluginId, "workspacePrepare", submit, rpcTimeout)); diff --git a/server/src/services/plugin-workspace-replay-context.ts b/server/src/services/plugin-workspace-replay-context.ts new file mode 100644 index 00000000000..76ff17e3157 --- /dev/null +++ b/server/src/services/plugin-workspace-replay-context.ts @@ -0,0 +1,22 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +type WorkspaceReplay = { + pluginId: string; + companyId: string; + projectId: string; + projectWorkspaceId: string; + executionWorkspaceId: string; +}; + +const replayContext = new AsyncLocalStorage(); + +export function withPluginWorkspaceReplay(scope: WorkspaceReplay, work: () => T): T { + return replayContext.run(scope, work); +} + +export function isPluginWorkspaceReplay(scope: WorkspaceReplay): boolean { + const replay = replayContext.getStore(); + return !!replay && replay.pluginId === scope.pluginId && replay.companyId === scope.companyId && + replay.projectId === scope.projectId && replay.projectWorkspaceId === scope.projectWorkspaceId && + replay.executionWorkspaceId === scope.executionWorkspaceId; +} diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index cab1346ed1d..67c9008b628 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -1,10 +1,12 @@ +import { isDeepStrictEqual } from "node:util"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertProjectStorageEnvironment, effectiveProjectWorkspacePolicy, requireWorkspaceProviderWorker, type WorkspaceWorkers } from "./project-workspace-provider.js"; -import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; +import { releaseProviderRecoveryTransfers } from "./provider-fresh-workspace.js"; import { retireAbandonedPluginWorkspaceReservation } from "./plugin-workspace-reservations.js"; import { randomUUID, createHash } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { + environmentLeases, executionWorkspaces, heartbeatRuns, issues, @@ -15,27 +17,45 @@ import { import { readWorkspaceProviderBinding, type WorkspaceProviderBinding, + type ExecutionWorkspace, } from "@paperclipai/shared"; import { + pluginWorkspaceParamsSchema, workspaceProviderOperationSchema, type PluginWorkspaceOperation, type PluginWorkspaceRecoveryPullRequest, type PluginWorkspaceParams, - type PluginWorkspaceResult, type PluginFreshWorkspace, } from "@paperclipai/plugin-sdk"; -import { invokeWorkspaceProvider } from "./plugin-workspace-preparation.js"; +import { callWorkspaceProvider } from "./plugin-workspace-preparation.js"; +import { withPluginWorkspaceReplay } from "./plugin-workspace-replay-context.js"; +import { pluginRegistryService } from "./plugin-registry.js"; import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; -import { WorkspaceProviderFailureError, workspaceProviderCallFailure } from "./workspace-provider-failure.js"; import { assertTaskWorkspaceExecutionAccess, assertTaskWorkspaceSourceProjectAccess, } from "./task-workspace-access.js"; import { forbidden, conflict } from "../errors.js"; +export { callWorkspaceProvider } from "./plugin-workspace-preparation.js"; export type { WorkspaceWorkers } from "./project-workspace-provider.js"; type Workspace = typeof executionWorkspaces.$inferSelect; +type TaskPlacement = NonNullable; +const preparationPlacementSchema = pluginWorkspaceParamsSchema.pick({ + generation: true, runId: true, agentId: true, taskLease: true, +}); + +function readPreparationPlacement(workspace: Workspace) { + const saved = workspace.metadata?.workspaceProviderPreparationPlacement; + if (saved === undefined) return undefined; + const placement = preparationPlacementSchema.parse(saved); + if (!placement.runId || !placement.agentId || + placement.generation !== (workspace.metadata?.lifecycleGeneration ?? 0)) + throw conflict("Workspace preparation placement changed"); + return placement; +} +type EnsureTaskWorkspace = (taskId: string, assertOwned: () => void) => Promise | undefined; export async function pluginWorkspaceSupportsMergeAncestry(db: Db, workers: WorkspaceWorkers | undefined, workspace: Workspace) { const pluginId = readWorkspaceProviderBinding(workspace.metadata)?.pluginId; @@ -44,102 +64,6 @@ export async function pluginWorkspaceSupportsMergeAncestry(db: Db, workers: Work return selected.manifestJson.workspaceProvider?.supportsMergeAncestry !== false; } -/** No paths or provider credentials are accepted from task callers. */ -export async function callWorkspaceProvider( - db: Db, - workers: WorkspaceWorkers | undefined, - input: { - workspace: Workspace; - operation: PluginWorkspaceOperation; - pluginId?: string; - advanceGeneration?: boolean; - runId?: string | null; - agentId?: string | null; - removalOperationId?: string; - }, -) { - const { workspace } = input; - const binding = readWorkspaceProviderBinding(workspace.metadata); - const pluginId = binding?.pluginId ?? input.pluginId; - if (!pluginId || !workspace.projectId) - throw conflict("Workspace storage ownership is missing"); - const selectedProvider = await requireWorkspaceProviderWorker(db, workers, workspace.companyId, pluginId, true); - const verifyExisting = input.operation.kind === "prepare" && input.operation.verifyExisting === true; - if (verifyExisting && selectedProvider.manifestJson.workspaceProvider?.workspaceVerificationVersion !== 1) - throw conflict("Workspace verification is unavailable"); - if (input.operation.kind === "prepare" && input.operation.fresh && selectedProvider.manifestJson.workspaceProvider?.freshWorkspaceVersion !== 1) - throw conflict("Fresh workspace preparation is unavailable"); - if (input.operation.kind === "finalize" && - (selectedProvider.manifestJson.workspaceProvider?.finalizationVersion !== 1 || !input.runId || !input.agentId)) - throw conflict("Workspace finalization is unavailable"); - const generation = - typeof workspace.metadata?.lifecycleGeneration === "number" - ? workspace.metadata.lifecycleGeneration - : 0; - const operationGeneration = generation + (input.advanceGeneration ? 1 : 0); - const operation = workspaceProviderOperationSchema.parse(input.operation); - if (input.removalOperationId !== undefined && (operation.kind !== "remove" || - input.removalOperationId !== createHash("sha256").update(JSON.stringify([workspace.id, operationGeneration, operation])).digest("hex"))) - throw conflict("Workspace removal identity does not match its recorded payload"); - if ((operation.kind === "inspect" || operation.kind === "remove") && input.removalOperationId === undefined - && selectedProvider.manifestJson.workspaceProvider?.recoveryPullRequestVersion !== 1) delete operation.recoveryPullRequest; - if (operation.kind === "prepare" && operation.fresh && !verifyExisting) { - if (!workspace.sourceIssueId) throw conflict("Fresh workspace task ownership is missing"); - operation.fresh = await stageProviderFreshWorkspace({ db, companyId: workspace.companyId, - issueId: workspace.sourceIssueId, ...operation.fresh, existing: operation.fresh }); - } - const params: Omit = { - pluginId, - companyId: workspace.companyId, - projectId: workspace.projectId, - executionWorkspaceId: workspace.id, - operationId: input.removalOperationId ?? ( - operation.kind === "inspect" - ? randomUUID() - : createHash("sha256") - .update( - JSON.stringify([workspace.id, operationGeneration, operation, - ...(operation.kind === "finalize" ? [input.runId, input.agentId] : [])]), - ) - .digest("hex")), - generation: operationGeneration, - runId: input.runId ?? null, - agentId: input.agentId ?? null, - operation, - }; - let result: PluginWorkspaceResult; - try { - result = await invokeWorkspaceProvider(workers!, params); - } catch (error) { - // A worker that never received the call, a lost reply and a provider-reported - // failure need different recovery; only a closed provider reason crosses over. - if (error instanceof WorkspaceProviderFailureError) throw error; - throw workspaceProviderCallFailure(error); - } - // Recheck enablement and ownership after an external effect, including an ambiguous retry. - await requireWorkspaceProviderWorker(db, workers, workspace.companyId, pluginId); - const [fresh] = await db - .select() - .from(executionWorkspaces) - .where( - and( - eq(executionWorkspaces.id, workspace.id), - eq(executionWorkspaces.companyId, workspace.companyId), - ), - ); - if ( - !fresh || - fresh.status !== workspace.status || - (fresh.metadata?.lifecycleGeneration ?? 0) !== generation || - fresh.providerRef !== workspace.providerRef - ) { - throw conflict("Workspace changed during storage preparation"); - } - if (result.kind === "prepared" && result.binding.pluginId !== pluginId) - throw conflict("Workspace provider ownership changed"); - return result; -} - export async function preparePluginWorkspace( db: Db, workers: WorkspaceWorkers | undefined, @@ -162,6 +86,8 @@ export async function preparePluginWorkspace( fresh?: PluginFreshWorkspace; expectedBindingRevision?: number; verifyExisting?: true; + acquireTaskLease?: (workspace: Pick) => Promise<{ leaseId: string; taskLease: TaskPlacement }>; + onTaskPrepared?: (ensure: EnsureTaskWorkspace) => void; }, ) { const [previous] = input.existingWorkspaceId @@ -260,11 +186,11 @@ export async function preparePluginWorkspace( ? null : input.responsibleUserId, }; - const authorize = async () => { + const authorize = async (workspaceId = existing?.id) => { await assertTaskWorkspaceExecutionAccess(db, actor, input.companyId, { issueId: input.issueId!, - ...(existing - ? { workspaceId: existing.id } + ...(workspaceId + ? { workspaceId } : { projectWorkspaceId: input.projectWorkspaceId }), }); await assertTaskWorkspaceSourceProjectAccess( @@ -417,7 +343,12 @@ export async function preparePluginWorkspace( .where(eq(issues.id, task.id)); return row!; })); - return withExecutionWorkspaceLifecycleLock(db, workspace.id, async (tx) => { + const driverKey = input.environment?.driver === "plugin" ? input.environment.config?.driverKey : undefined; + const placement = selectedProvider.manifestJson.environmentDrivers?.some(driver => + driver.driverKey === driverKey && driver.sessionHandoffVersion === 1) + ? await input.acquireTaskLease?.({ id: workspace.id, mode: mode as ExecutionWorkspace["mode"] }) : undefined; + const prepare = (workspace: Workspace, taskLease?: TaskPlacement, replacement?: { leaseId: string; assertOwned: () => void }) => withExecutionWorkspaceLifecycleLock(db, workspace.id, async (tx) => { + replacement?.assertOwned(); const [current] = await tx .select() .from(executionWorkspaces) @@ -433,17 +364,50 @@ export async function preparePluginWorkspace( ) { throw conflict("Workspace lifecycle changed; reopen before preparing"); } + if (taskLease) await authorize(replacement ? workspace.id : undefined); + const replacementScope = replacement && taskLease ? and( + eq(environmentLeases.id, replacement.leaseId), + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), + eq(environmentLeases.executionWorkspaceId, workspace.id), + eq(environmentLeases.environmentId, taskLease.environmentId), + eq(environmentLeases.providerLeaseId, taskLease.providerLeaseId), + eq(environmentLeases.status, "active"), + sql`${environmentLeases.metadata}->>'pluginId' = ${pluginId} and ${environmentLeases.metadata}->>'driverKey' = ${taskLease.driverKey}`, + sql`${environmentLeases.releasedAt} is null and (${environmentLeases.expiresAt} is null or ${environmentLeases.expiresAt} > clock_timestamp())`, + sql`coalesce(${environmentLeases.metadata}->>'runnerTaskRetiring', 'false') != 'true'`, + ) : undefined; + if (replacementScope && !(await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(replacementScope)).length) + throw conflict("Workspace Launcher placement changed"); + if (!current.providerRef && !input.verifyExisting) { + const generation = current.metadata?.lifecycleGeneration ?? 0; + const intent = { generation, runId: input.runId, agentId: input.agentId, ...(taskLease ? { taskLease } : {}) }; + const previous = readPreparationPlacement(current); + if (!isDeepStrictEqual(previous, intent)) { + // The request must survive a lost reply or rollback of the lifecycle transaction. + const [recorded] = await db.update(executionWorkspaces).set({ + metadata: sql`coalesce(${executionWorkspaces.metadata}, '{}'::jsonb) || ${JSON.stringify({ workspaceProviderPreparationPlacement: intent })}::jsonb`, + }).where(and(eq(executionWorkspaces.id, current.id), eq(executionWorkspaces.companyId, current.companyId), + sql`${executionWorkspaces.providerRef} is null + and coalesce((${executionWorkspaces.metadata}->>'lifecycleGeneration')::int, 0) = ${generation} + and coalesce(${executionWorkspaces.metadata}->'workspaceProviderPreparationPlacement', 'null'::jsonb) + = ${JSON.stringify(previous ?? null)}::jsonb`)).returning({ id: executionWorkspaces.id }); + if (!recorded) throw conflict("Workspace preparation placement changed"); + } + } const result = await callWorkspaceProvider(tx as unknown as Db, workers, { workspace, pluginId, runId: input.runId, agentId: input.agentId, + taskLease, + assertOwned: replacement?.assertOwned, operation: { kind: "prepare", source: sourceInput, mode: workspace.mode as "isolated_workspace", strategy: workspace.strategyType as "git_worktree", - branchName: workspace.metadata?.workspaceProviderFresh ? null : branchName, + branchName: workspace.metadata?.workspaceProviderFresh ? null : workspace.branchName, branchMode: (workspace.metadata?.workspaceProviderBranchMode as | "create" @@ -452,13 +416,15 @@ export async function preparePluginWorkspace( input.branchMode ?? "create", baseRef: workspace.baseRef, - existing: saved, + existing: readWorkspaceProviderBinding(workspace.metadata), ...(input.verifyExisting ? { verifyExisting: true } : {}), ...(workspace.metadata?.workspaceProviderFresh ? { fresh: workspace.metadata.workspaceProviderFresh as PluginFreshWorkspace } : {}), }, }); if (result.kind !== "prepared") throw conflict("Workspace preparation is incomplete"); + if (replacement && result.branchName !== workspace.branchName) + throw conflict("Workspace Launcher prepared a different branch"); const savedBase = workspace.metadata?.workspaceProviderBaseSha; if (input.verifyExisting && (result.created || !result.baseSha || (saved && result.binding.workspaceRef !== saved.workspaceRef) || @@ -479,7 +445,14 @@ export async function preparePluginWorkspace( .where(eq(issues.id, input.issueId!)); if (task?.executionWorkspaceId !== workspace.id) throw conflict("Task workspace binding changed during preparation"); + if (replacementScope) { + const [currentLease] = await tx.update(environmentLeases).set({ + metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ workspaceProvider: result.binding })}::jsonb`, + }).where(replacementScope).returning({ id: environmentLeases.id }); + if (!currentLease) throw conflict("Workspace Launcher placement changed during preparation"); + } await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`); + const { workspaceProviderPreparationPlacement: _placement, ...metadata } = current.metadata ?? {}; const [updated] = await tx .update(executionWorkspaces) .set({ @@ -489,7 +462,7 @@ export async function preparePluginWorkspace( lastUsedAt: new Date(), updatedAt: new Date(), metadata: { - ...workspace.metadata, + ...metadata, workspaceProvider: result.binding, workspaceProviderBaseSha: result.baseSha, ...(result.destinationRef !== undefined ? { workspaceProviderDestinationRef: result.destinationRef } : {}), @@ -511,6 +484,19 @@ export async function preparePluginWorkspace( throw conflict("Workspace changed before preparation could be recorded"); return updated!; }); + const prepared = await prepare(workspace, placement?.taskLease); + if (placement && input.onTaskPrepared) { + let preparedTaskId = placement.taskLease.providerLeaseId; + let preparedWorkspace = prepared; + input.onTaskPrepared((taskId, assertOwned) => { + if (taskId === preparedTaskId) return; + return prepare(preparedWorkspace, { ...placement.taskLease, providerLeaseId: taskId }, { leaseId: placement.leaseId, assertOwned }).then(updated => { + preparedWorkspace = updated; + preparedTaskId = taskId; + }); + }); + } + return prepared; } /** Recover a lost prepare result even when the originating run has already stopped. */ @@ -534,7 +520,7 @@ async function reconcilePluginWorkspaceReservation( const owner = workspace.metadata?.workspaceProviderOwner; const retired = typeof owner === "string" ? await retireAbandonedPluginWorkspaceReservation(db, workspace) : null; if (retired) return retired; - if (typeof owner !== "string" || !workspace.metadata?.workspaceProviderSource) + if (typeof owner !== "string" || !workspace.projectId || !workspace.metadata?.workspaceProviderSource) throw conflict("Workspace storage reservation is unavailable"); const operation = workspaceProviderOperationSchema.parse({ kind: "prepare", @@ -547,13 +533,19 @@ async function reconcilePluginWorkspaceReservation( existing: null, ...(workspace.metadata?.workspaceProviderFresh ? { fresh: workspace.metadata.workspaceProviderFresh as PluginFreshWorkspace } : {}), }); - const result = await callWorkspaceProvider(db, workers, { + if (operation.kind !== "prepare") throw conflict("Workspace preparation is unavailable"); + const result = await withPluginWorkspaceReplay({ + pluginId: owner, companyId: workspace.companyId, projectId: workspace.projectId, + projectWorkspaceId: operation.source.id, executionWorkspaceId: workspace.id, + }, () => callWorkspaceProvider(db, workers, { workspace, pluginId: owner, + ...readPreparationPlacement(workspace), operation, - }); + })); if (result.kind !== "prepared") throw conflict("Workspace reconciliation is incomplete"); + const { workspaceProviderPreparationPlacement: _placement, ...metadata } = workspace.metadata ?? {}; const [updated] = await db.transaction(async (tx) => { await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`); return tx @@ -563,7 +555,7 @@ async function reconcilePluginWorkspaceReservation( branchName: result.branchName, updatedAt: new Date(), metadata: { - ...workspace.metadata, + ...metadata, workspaceProvider: result.binding, workspaceProviderBaseSha: result.baseSha, ...(result.destinationRef !== undefined ? { workspaceProviderDestinationRef: result.destinationRef } : {}), @@ -621,20 +613,58 @@ export async function inspectPluginWorkspace(db: Db, workers: WorkspaceWorkers | ? Promise.resolve(removedInspection()) : inspectBinding(db, workers, current, recoveryPullRequest)); } +type PluginWorkspaceFinalizationPlan = { kind: "inspect" } | { kind: "finalize"; taskLease?: TaskPlacement }; + +export async function resolvePluginWorkspaceFinalization(db: Db, workspace: Workspace, + context: { runId: string; agentId: string; taskLease?: TaskPlacement | null }): Promise { + if (workspace.providerType !== "plugin") return undefined; + if (context.taskLease) return { kind: "finalize", taskLease: context.taskLease }; + const binding = readWorkspaceProviderBinding(workspace.metadata); + if (!binding) throw conflict("Workspace storage ownership is missing"); + const selected = await pluginRegistryService(db).getById(binding.pluginId); + if (!selected) throw conflict("The workspace storage plugin is unavailable"); + // Existing providers continue to use their established inspection contract. + if (selected.manifestJson.workspaceProvider?.finalizationVersion !== 1) return { kind: "inspect" }; + const portableDrivers = new Set(selected.manifestJson.environmentDrivers?.filter(driver => + driver.sessionHandoffVersion === 1).map(driver => driver.driverKey)); + let taskLease: TaskPlacement | undefined; + if (context.taskLease === undefined && portableDrivers.size > 0) { + const leases = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, workspace.companyId), + eq(environmentLeases.executionWorkspaceId, workspace.id), + eq(environmentLeases.heartbeatRunId, context.runId), + eq(environmentLeases.status, "active"), + )); + const placements = leases.flatMap(lease => { + const driverKey = lease.metadata?.driverKey; + return lease.metadata?.pluginId === binding.pluginId && lease.metadata?.agentId === context.agentId && + lease.metadata?.runnerTaskLease === true && lease.environmentId && lease.providerLeaseId && + typeof driverKey === "string" && portableDrivers.has(driverKey) + ? [{ driverKey, environmentId: lease.environmentId, providerLeaseId: lease.providerLeaseId }] : []; + }); + if (placements.length > 1) throw conflict("Workspace Launcher placement is ambiguous"); + taskLease = placements[0]; + } + return { kind: "finalize", taskLease }; +} + /** Completion does not need the deletion-safety proof supplied by inspect. */ export async function finalizePluginWorkspace(db: Db, workers: WorkspaceWorkers | undefined, - workspace: Workspace, context: { runId: string; agentId: string }) { + workspace: Workspace, context: { runId: string; agentId: string; taskLease?: TaskPlacement | null; + plan?: PluginWorkspaceFinalizationPlan }) { return withRetainedWorkspace(db, workers, workspace, async (current) => { if (current.metadata?.workspaceProviderRemoved === true) throw conflict("Workspace was removed before finalization"); const binding = readWorkspaceProviderBinding(current.metadata); if (!binding) throw conflict("Workspace storage ownership is missing"); - const selected = await requireWorkspaceProviderWorker(db, workers, current.companyId, binding.pluginId); - // Existing providers continue to use their established inspection contract. - if (selected.manifestJson.workspaceProvider?.finalizationVersion !== 1) - return inspectBinding(db, workers, current); + if (context.plan && !isDeepStrictEqual(binding, readWorkspaceProviderBinding(workspace.metadata))) + throw conflict("Workspace storage ownership changed"); + const plan = context.plan ?? await resolvePluginWorkspaceFinalization(db, current, context); + if (!plan) throw conflict("The workspace storage plugin is unavailable"); + if (plan.kind === "inspect") return inspectBinding(db, workers, current); + const taskLease = plan.taskLease; const result = await callWorkspaceProvider(db, workers, { - workspace: current, ...context, operation: { kind: "finalize", binding }, + workspace: current, runId: context.runId, agentId: context.agentId, taskLease, operation: { kind: "finalize", binding }, }); if (result.kind !== "finalized") throw conflict("Workspace finalization is incomplete"); return result; diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index dfa48f22284..2623962bee4 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -31,7 +31,7 @@ "packages/plugins/sdk/src/ui/types.ts": "5a5644acf6554d2e5e1d000bfd03b97d57001eb2", "packages/plugins/sdk/src/worker-rpc-host.ts": "3de7748e983ad1a749a7a7f62f685dab4399cdd1", "packages/plugins/sdk/src/workspace-operation-error.ts": "2cd247d002d83b163239229e631bca4e519e5075", - "packages/plugins/sdk/src/workspace-provider.ts": "dea90b92e5bf0c05062eb57598c70f370e207cea", + "packages/plugins/sdk/src/workspace-provider.ts": "bdfe5e91d4569df0a5d31e1bfc200aff1bf68f5b", "packages/plugins/sdk/tsconfig.json": "d62ebdff94bc4b75dabf1fcad187e1e7a90fc794", "packages/shared/package.json": "6417ef8baef0cb9b836b14f453dc8bc5025709e0", "packages/shared/src/__snapshots__/slack-app-manifest.test.ts.snap": "79020ba3aa6662923f70aa44e2c483844e9901d9",