diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 68cc532113b..b518f32a95b 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1641,8 +1641,15 @@ Retained-file recovery has a separate, source-bound maintenance path. It uses the existing physical copyback lock and durable controller receipt. It verifies the exact stopped allocation and immutable workspace descriptor, resumes only that allocation for export, preserves newer host edits through the normal merge, -and obtains a fresh stop receipt. An interrupted legacy run does not need an -accepted native result. A successful `workspace_recovery` operation and matching +and obtains a fresh stop receipt. The owned file-recovery path can request the +exact recorded directory through the driver-only `requestedCwd` resume field. +Daytona requires that field to equal the recorded lease directory and verifies +the allocation sentinel at its recorded path. It returns the requested directory +without creating a folder. Ordinary resume omits the field and returns the +allocation root for normal workspace preparation. Recovery still requires the +reopened allocation and directory to match the recorded source exactly. +An interrupted legacy run does not need an accepted native result. A successful +`workspace_recovery` operation and matching recovery record prove persistence; adapter output or a user repair note cannot. The failed run and unknown external effects remain historical facts. Cancelled native work can owe files after an audited run-only Stop from the diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index b7cd83089b9..95a725cf508 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -2147,6 +2147,112 @@ describe("Daytona sandbox provider plugin", () => { }); }); + it.each([true, false])("resumes the recorded workspace cwd after workspace preparation (reuse=%s)", async reuseLease => { + const sandbox = createMockSandbox({ id: "sandbox-retained", state: "stopped" }); + const allocationRoot = "/home/daytona/paperclip-workspace"; + const workspaceId = "6028a46b-b65a-421f-952d-66b9c002b8e2"; + const remoteCwd = `${allocationRoot}/.paperclip-workspaces/${workspaceId}`; + const sentinelPath = `${allocationRoot}/.paperclip-runtime/reusable-sandbox-lease.json`; + sandbox.process.executeCommand.mockImplementation(async command => ({ exitCode: 0, + result: command.startsWith("cat ") ? JSON.stringify({ token: "sentinel-token" }) : "bash" })); + mockGet.mockResolvedValue(sandbox); + const resumed = await plugin.definition.onEnvironmentResumeLease!({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandbox.id, + config: { apiKey: "fixture-key", reuseLease }, requestedCwd: remoteCwd, leaseMetadata: { remoteCwd, remoteWorkspaceId: workspaceId, + sandboxId: sandbox.id, reuseLease, workspaceSentinel: { path: sentinelPath, token: "sentinel-token", + runId: "fixture-run", providerLeaseId: sandbox.id, result: "written" } }, + }); + expect(resumed).toMatchObject({ providerLeaseId: sandbox.id, metadata: { + remoteCwd, workspaceSentinel: { path: sentinelPath, result: "matched" } } }); + expect(sandbox.process.executeCommand.mock.calls[0][0]).toBe(`cat '${sentinelPath}'`); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + expect(sandbox.fs.uploadFile).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + }); + + it.each([null, 7, "", "relative/workspace", "/workspace/../other", "/workspace\0other"])( + "rejects a malformed recorded cwd before touching the provider: %j", async remoteCwd => { + await expect(plugin.definition.onEnvironmentResumeLease!({ driverKey: "daytona", companyId: "company-1", + environmentId: "env-1", providerLeaseId: "sandbox-retained", config: { apiKey: "fixture-key" }, + requestedCwd: remoteCwd as string, leaseMetadata: { remoteCwd }, + })).rejects.toThrow("recorded Daytona workspace directory"); + expect(mockGet).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }, + ); + + it("retains a nested workspace only after the allocation sentinel matches", async () => { + const sandbox = createMockSandbox({ state: "stopped" }); + sandbox.process.executeCommand.mockResolvedValue({ exitCode: 0, result: JSON.stringify({ token: "foreign" }) }); + mockGet.mockResolvedValue(sandbox); + const resumed = await plugin.definition.onEnvironmentResumeLease!({ driverKey: "daytona", companyId: "company-1", + environmentId: "env-1", providerLeaseId: sandbox.id, config: { apiKey: "fixture-key" }, + requestedCwd: "/home/daytona/paperclip-workspace/.paperclip-workspaces/fixture-workspace", leaseMetadata: { + remoteCwd: "/home/daytona/paperclip-workspace/.paperclip-workspaces/fixture-workspace", + workspaceSentinel: { path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + token: "expected", result: "written" }, + } }); + expect(resumed).toMatchObject({ providerLeaseId: null, metadata: { expired: true, workspaceSentinel: { result: "mismatch" } } }); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + expect(sandbox.fs.uploadFile).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each(["same-workspace", "later-task-workspace"])("keeps the allocation root for ordinary %s admission", async workspaceId => { + const sandbox = createMockSandbox({ state: "stopped" }); + const allocationRoot = "/home/daytona/paperclip-workspace"; + sandbox.process.executeCommand.mockImplementation(async command => ({ exitCode: 0, + result: command.startsWith("cat ") ? JSON.stringify({ token: "sentinel-token" }) : "bash" })); + mockGet.mockResolvedValue(sandbox); + const resumed = await plugin.definition.onEnvironmentResumeLease!({ driverKey: "daytona", companyId: "company-1", + environmentId: "env-1", providerLeaseId: sandbox.id, config: { apiKey: "fixture-key" }, leaseMetadata: { + remoteCwd: `${allocationRoot}/.paperclip-workspaces/${workspaceId}`, + workspaceSentinel: { path: `${allocationRoot}/.paperclip-runtime/reusable-sandbox-lease.json`, token: "sentinel-token" }, + } }); + expect(resumed).toMatchObject({ providerLeaseId: sandbox.id, metadata: { remoteCwd: allocationRoot } }); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each([undefined, null, "/home/daytona/paperclip-workspace/.paperclip-workspaces/foreign"])( + "rejects an export cwd without an exact recorded source: %j", async recordedCwd => { + await expect(plugin.definition.onEnvironmentResumeLease!({ driverKey: "daytona", companyId: "company-1", + environmentId: "env-1", providerLeaseId: "sandbox-retained", config: { apiKey: "fixture-key" }, + requestedCwd: "/home/daytona/paperclip-workspace/.paperclip-workspaces/owned", + leaseMetadata: recordedCwd === undefined ? {} : { remoteCwd: recordedCwd }, + })).rejects.toThrow("recorded Daytona workspace directory"); + expect(mockGet).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }, + ); + + it("opens existing files in the recorded directory after a stopped sandbox resumes", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "daytona-resume-cwd-")); + const allocationRoot = path.join(directory, "paperclip-workspace"); + const remoteCwd = path.join(allocationRoot, ".paperclip-workspaces", "fixture-workspace"); + const sentinelPath = path.join(allocationRoot, ".paperclip-runtime", "reusable-sandbox-lease.json"); + await fs.mkdir(remoteCwd, { recursive: true }); + await fs.mkdir(path.dirname(sentinelPath), { recursive: true }); + await fs.writeFile(sentinelPath, JSON.stringify({ token: "file-token" })); + await fs.writeFile(path.join(remoteCwd, "saved.txt"), "retained work\n"); + const sandbox = createMockSandbox({ state: "stopped", workDir: directory }); + sandbox.process.executeCommand.mockImplementation(async command => ({ exitCode: 0, + result: execFileSync("sh", ["-c", command], { encoding: "utf8", cwd: directory }) })); + mockGet.mockResolvedValue(sandbox); + try { + const resumed = await plugin.definition.onEnvironmentResumeLease!({ driverKey: "daytona", companyId: "company-1", + environmentId: "env-1", providerLeaseId: sandbox.id, config: { apiKey: "fixture-key" }, requestedCwd: remoteCwd, + leaseMetadata: { remoteCwd, workspaceSentinel: { path: sentinelPath, token: "file-token" } } }); + expect(await fs.readFile(path.join(resumed.metadata!.remoteCwd as string, "saved.txt"), "utf8")).toBe("retained work\n"); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + expect(sandbox.fs.uploadFile).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockGet).toHaveBeenCalledWith(sandbox.id); + } finally { await fs.rm(directory, { recursive: true, force: true }); } + }); + describe("missing-container resume", () => { const sandboxId = "00000000-0000-4000-8000-000000000001"; const missing = `not found: failed to inspect sandbox container ${sandboxId}: Error response from daemon: No such container: ${sandboxId}`; diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 45ba1de804f..01bfa8580d0 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -2780,6 +2780,12 @@ const plugin = definePlugin(trackDaytonaHooks({ params: PluginEnvironmentResumeLeaseParams, ): Promise { const config = parseDriverConfig(params.config); + const requestedCwd = params.requestedCwd; + if (requestedCwd !== undefined && (typeof requestedCwd !== "string" || requestedCwd.includes("\0") + || !path.posix.isAbsolute(requestedCwd) || path.posix.normalize(requestedCwd) !== requestedCwd + || params.leaseMetadata?.remoteCwd !== requestedCwd)) { + throw new Error("The requested directory does not match the recorded Daytona workspace directory."); + } if (params.leaseMetadata?.reuseLease === false) { const sentinel = isRecord(params.leaseMetadata.workspaceSentinel) ? params.leaseMetadata.workspaceSentinel : null; const intent = isRecord(params.leaseMetadata.nativeWorkspaceExportResume) ? params.leaseMetadata.nativeWorkspaceExportResume : null; @@ -2877,7 +2883,7 @@ const plugin = definePlugin(trackDaytonaHooks({ config, sandbox, shellCommand, - remoteCwd, + remoteCwd: requestedCwd ?? remoteCwd, resumedLease: true, resumedFromState, sandboxState: "started", diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index b308c1287c4..77189311219 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -677,6 +677,7 @@ export interface PluginEnvironmentResumeLeaseParams extends PluginEnvironmentDri runId?: string; providerLeaseId: string; leaseMetadata?: Record; + requestedCwd?: string; } export interface PluginEnvironmentReleaseLeaseParams extends PluginEnvironmentDriverBaseParams { diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 324a0eda702..01c9ec1cb19 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -49,6 +49,7 @@ import { logger } from "../middleware/logger.ts"; import { agentEnvironmentTestService } from "../services/agent-environment-test.js"; import { resolveRunnerEnvironmentForRun } from "../services/runner-environment-lifecycle.js"; import { environmentService } from "../services/environments.ts"; +import { resumePluginEnvironmentLease } from "../services/plugin-environment-driver.js"; import { remoteExecutionHasStopped } from "../services/remote-execution-termination.ts"; import { heartbeatService } from "../services/heartbeat.ts"; import { secretService } from "../services/secrets.ts"; @@ -3810,7 +3811,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); await environmentService(db).releaseLease(seeded.reusableLease.id, "released", { cleanupStatus: "success" }); const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!; - const call = vi.fn(async (_id: string, method: string) => { + const call = vi.fn(async (_id: string, method: string, _input?: unknown) => { if (method !== "environmentResumeLease") throw new Error("Only exact resume is permitted"); return { providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } }; }); @@ -3824,10 +3825,27 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { companyId: seeded.companyId, environmentId: seeded.environment.id, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata, }), expect.any(Number)); + expect(call.mock.calls[0]?.[2]).not.toHaveProperty("requestedCwd"); expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("released"); expect(await db.select().from(environmentLeases)).toHaveLength(1); }); + it("forwards an explicit export directory to the selected generic plugin", async () => { + const seeded = await seedReusablePluginSandboxLease(); + const requestedCwd = `/workspace/.paperclip-workspaces/${seeded.executionWorkspaceId}`; + const metadata = { ...seeded.reusableLease.metadata, remoteCwd: requestedCwd }; + const call = vi.fn(async () => ({ providerLeaseId: seeded.reusableLease.providerLeaseId!, metadata })); + const workerManager = { isRunning: () => true, call } as unknown as PluginWorkerManager; + const result = await resumePluginEnvironmentLease({ db, workerManager, companyId: seeded.companyId, + environmentId: seeded.environment.id, config: { pluginKey: "acme.reusable-sandbox-provider", driverKey: "fake-plugin", driverConfig: {} }, + providerLeaseId: seeded.reusableLease.providerLeaseId!, leaseMetadata: metadata, requestedCwd }); + expect(result.metadata?.remoteCwd).toBe(requestedCwd); + expect(call).toHaveBeenCalledExactlyOnceWith(seeded.pluginId, "environmentResumeLease", expect.objectContaining({ + companyId: seeded.companyId, environmentId: seeded.environment.id, + providerLeaseId: seeded.reusableLease.providerLeaseId, leaseMetadata: metadata, requestedCwd, + })); + }); + it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease; diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 9de7ae9c2d9..30acbbe1db5 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -541,6 +541,10 @@ export interface EnvironmentDriverLeaseInput { failureReason?: string; } +export interface EnvironmentDriverResumeInput extends EnvironmentDriverLeaseInput { + requestedCwd?: string; +} + export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput { workspace: { localPath?: string; @@ -612,7 +616,7 @@ export interface EnvironmentRuntimeDriver { inspectLeaseResource?(input: { environment: Environment | null; lease: EnvironmentLease }): Promise; acquireRunLease(input: EnvironmentDriverAcquireInput): Promise; releaseRunLease(input: EnvironmentDriverReleaseInput): Promise; - resumeRunLease?(input: EnvironmentDriverLeaseInput): Promise; + resumeRunLease?(input: EnvironmentDriverResumeInput): Promise; destroyRunLease?(input: EnvironmentDriverLeaseInput): Promise; realizeWorkspace?(input: EnvironmentDriverRealizeWorkspaceInput): Promise; execute?(input: EnvironmentDriverExecuteInput): Promise; @@ -2577,6 +2581,7 @@ function createSandboxEnvironmentDriver( driverKey: providerKey, companyId: input.lease.companyId, environmentId: input.environment.id, issueId: input.lease.issueId, config, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, + ...(input.requestedCwd !== undefined ? { requestedCwd: input.requestedCwd } : {}), }, Math.min(resolvePluginSandboxRpcTimeoutMs(config) ?? 60_000, 60_000)); if (resumed?.providerLeaseId !== input.lease.providerLeaseId) { throw new Error("The provider did not confirm the exact retained sandbox. No replacement was acquired."); @@ -3413,6 +3418,7 @@ function createPluginEnvironmentDriver( }, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, + ...(input.requestedCwd !== undefined ? { requestedCwd: input.requestedCwd } : {}), }); }, @@ -4305,13 +4311,14 @@ export function environmentRuntimeService( source: WorkspaceRecoverySourceIdentity; leaseId: string; environmentId: string; ownership: NativeWorkspaceFinalizationOwnership; }): Promise { - const { lease, environment } = await ownedWorkspaceRecoveryLease(db, { ...input.source, + const { lease, environment, remoteCwd } = await ownedWorkspaceRecoveryLease(db, { ...input.source, leaseId: input.leaseId, environmentId: input.environmentId, ownership: input.ownership }); const driver = requireDriverKey(getLeaseDriverKey(lease, environment as Environment)); if (!driver.resumeRunLease) throw new Error(`Environment driver "${driver.driver}" does not support lease resume.`); // Maintenance reopens the protected source only to preserve its files. // Ordinary resume and destructive cleanup keep their existing fences. - return await driver.resumeRunLease({ environment: environment as Environment, lease: lease as EnvironmentLease }); + return await driver.resumeRunLease({ environment: environment as Environment, lease: lease as EnvironmentLease, + requestedCwd: remoteCwd }); }, async destroyRunLease(input: EnvironmentDriverLeaseInput): Promise { diff --git a/server/src/services/plugin-environment-driver.ts b/server/src/services/plugin-environment-driver.ts index 68fafdca165..3e39c7dcc3e 100644 --- a/server/src/services/plugin-environment-driver.ts +++ b/server/src/services/plugin-environment-driver.ts @@ -527,6 +527,7 @@ export async function resumePluginEnvironmentLease(input: { config: PluginEnvironmentConfig; providerLeaseId: string; leaseMetadata?: Record; + requestedCwd?: string; }): Promise { const { plugin } = await resolvePluginEnvironmentDriver(input); return await input.workerManager.call(plugin.id, "environmentResumeLease", { @@ -537,6 +538,7 @@ export async function resumePluginEnvironmentLease(input: { config: input.config.driverConfig, providerLeaseId: input.providerLeaseId, leaseMetadata: input.leaseMetadata, + ...(input.requestedCwd !== undefined ? { requestedCwd: input.requestedCwd } : {}), }); } diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 5b1a0b19c5b..8648acad550 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -65,11 +65,13 @@ import { hasVerifiedWorkspaceRepair, readWorkspaceRecovery } from "./workspace-r import { claimNativeSessionResumptions } from "./native-runtime/native-finalization-reconciler.js"; import { executionWorkspaces } from "@paperclipai/db"; import { executionWorkspaceSnapshots } from "@paperclipai/db"; -import { createSavedWorkspaceLoader, readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "./remote-workspaces.js"; +import { createSavedWorkspaceLoader, prepareRemoteWorkspace, readRemoteWorkspaceAuthority, reserveRemoteWorkspace, saveRemoteWorkspace } from "./remote-workspaces.js"; import { createReusableLeaseHandoff } from "./environment-lease-handoff.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js"; import { getExecutionBlocker } from "./execution-blocker.js"; import { issueRoutes } from "../routes/issues.js"; +import { plugins } from "@paperclipai/db"; +import type { PluginWorkerManager } from "./plugin-worker-manager.js"; import { errorHandler } from "../middleware/index.js"; import * as workerProof from "./native-runtime/native-workspace-platform-proof.js"; const targets = vi.hoisted(() => new Map()); @@ -130,7 +132,7 @@ describe("automatic retained workspace recovery", () => { if (mode === "legacy") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "codex_local" } } }).where(eq(heartbeatRuns.id, runId)); if (mode === "native") await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "terminal_failure" }); const [environment] = await db.insert(environments).values({ name: runId, driver: "sandbox" }).returning(); - const providerLeaseId = randomUUID(), remote = path.join(root, runId, "remote"), host = path.join(root, runId, "host"); + const providerLeaseId: string = randomUUID(), remote = path.join(root, runId, "remote"), host = path.join(root, runId, "host"); await mkdir(host, { recursive: true }); await writeFile(path.join(host, "existing.txt"), "base\n"); const leaseValues = { companyId, issueId, heartbeatRunId: runId, environmentId: environment.id, provider: "daytona", providerLeaseId, @@ -1127,6 +1129,7 @@ describe("automatic retained workspace recovery", () => { expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); expect(f.runtime.resumeRunLease.mock.calls.filter(([input]) => input.lease.id === f.lease.id)).toHaveLength(1); + expect(f.runtime.resumeRunLease).toHaveBeenCalledWith(expect.objectContaining({ requestedCwd: f.remote })); await recoverRetainedWorkspace({ ...f.input, environmentRuntime: runtime }); expect(f.runtime.resumeRunLease.mock.calls.filter(([input]) => input.lease.id === f.lease.id)).toHaveLength(1); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); @@ -1157,6 +1160,92 @@ describe("automatic retained workspace recovery", () => { expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); }, 30_000); + it("forwards only the owned export directory through the sandbox provider boundary", async () => { + const driver = "sandbox"; + const f = await seed(); + const pluginId = f.lease.metadata!.pluginId as string; + const manifest: typeof plugins.$inferInsert.manifestJson = { id: `fixture.export-cwd.${pluginId}`, apiVersion: 1, version: "1.0.0", + displayName: "Fixture export provider", description: "Records provider boundary calls", author: "Paperclip", + categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [{ driverKey: "daytona", kind: "sandbox_provider", displayName: "Fixture", configSchema: { type: "object" } }] }; + await db.insert(plugins).values({ id: pluginId, pluginKey: manifest.id, + packageName: "@fixture/export-cwd", version: manifest.version, + apiVersion: 1, categories: manifest.categories, manifestJson: manifest, status: "ready", installOrder: 1 }); + const metadata = { ...f.lease.metadata, driver, provider: "daytona", pluginKey: manifest.id, + sandboxProviderPlugin: true }; + await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environments).set({ driver, config: { provider: "daytona" } }).where(eq(environments.id, f.lease.environmentId!)); + const call = vi.fn(async (_id: string, method: string, input: Record) => { + if (method === "environmentResumeLease") return { providerLeaseId: f.providerLeaseId, + metadata: { remoteCwd: input.requestedCwd ?? path.dirname(f.remote) } }; + if (method === "environmentStopLease") return { providerLeaseId: f.providerLeaseId, state: "stopped" }; + throw new Error(`Unexpected provider operation: ${method}`); + }); + const worker = { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentStopLease"] }) } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker }); + await recoverRetainedWorkspace({ ...f.input, environmentRuntime: runtime }); + expect(readWorkspaceRecovery((await action(f.actionId)).evidence)?.status).toBe("recovered"); + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease")).toEqual([ + expect.arrayContaining([pluginId, "environmentResumeLease", expect.objectContaining({ + companyId: f.companyId, environmentId: f.lease.environmentId, providerLeaseId: f.providerLeaseId, requestedCwd: f.remote, + })]), + ]); + expect(call.mock.calls.some(([, method]) => ["environmentAcquireLease", "environmentRealizeWorkspace", "environmentDestroyLease"].includes(method))).toBe(false); + expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + }, 30_000); + + it.each(["same", "later"])("prepares the %s task workspace from an ordinary resumed allocation root", async kind => { + const f = await seed("native", false); + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); + try { + const allocationRoot = await realpath(f.remote); + const target = { ...targets.get(f.lease.id)!, remoteCwd: allocationRoot }; + const storage = { ...getStorageService(), provider: "s3" as const }; + const agentId = (await run(f.runId)).agentId; + await db.update(heartbeatRuns).set({ status: "running", resultJson: {} }).where(eq(heartbeatRuns.id, f.runId)); + const [firstLease] = await db.update(environmentLeases).set({ status: "active", metadata: { + ...f.lease.metadata, remoteCwd: allocationRoot } }).where(eq(environmentLeases.id, f.lease.id)).returning(); + const reserveInput = { db, storage, companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId, + environmentId: f.lease.environmentId!, projectId: null, projectWorkspaceId: null, existingWorkspaceId: null, + source: { kind: "empty" as const } }; + const first = await prepareRemoteWorkspace({ db, storage, workspace: await reserveRemoteWorkspace(reserveInput), + runId: f.runId, lease: firstLease as EnvironmentLease, target }); + await writeFile(path.join(first.remoteRoot, "saved.txt"), "prior task work\n"); + await saveRemoteWorkspace({ db, storage, workspace: first.workspace, runId: f.runId, lease: first.lease, + target, assertOwned: async () => undefined }); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.id, f.lease.id)); + const issueId = kind === "same" ? f.issueId : randomUUID(); + if (kind === "later") await db.insert(issues).values({ id: issueId, companyId: f.companyId, assigneeAgentId: agentId, + title: "A later task", status: "in_progress" }); + const [nextRun] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId, issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "running", contextSnapshot: { issueId } }).returning(); + const [nextLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId, heartbeatRunId: nextRun.id, + environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: kind === "same" ? f.providerLeaseId : randomUUID(), + status: "active", metadata: { ...f.lease.metadata, remoteCwd: first.remoteRoot } }).returning(); + f.runtime.resumeRunLease.mockImplementation(async input => ({ providerLeaseId: input.lease.providerLeaseId!, + metadata: { remoteCwd: allocationRoot } })); + const [environment] = await db.select().from(environments).where(eq(environments.id, f.lease.environmentId!)); + const resumed = await f.environmentRuntime.resumeRunLease({ environment: environment as never, lease: nextLease as EnvironmentLease }); + expect(resumed?.metadata?.remoteCwd).toBe(allocationRoot); + expect(f.runtime.resumeRunLease.mock.calls[0]?.[0]).not.toHaveProperty("requestedCwd"); + const nextWorkspace = await reserveRemoteWorkspace({ ...reserveInput, issueId, runId: nextRun.id, + existingWorkspaceId: kind === "same" ? first.workspace.id : null }); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: nextWorkspace, runId: nextRun.id, + lease: nextLease as EnvironmentLease, target: { ...target, remoteCwd: resumed!.metadata!.remoteCwd as string } }); + expect(prepared.remoteRoot).toBe(path.join(allocationRoot, ".paperclip-workspaces", nextWorkspace.id)); + if (kind === "same") expect(await readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("prior task work\n"); + else expect(prepared.remoteRoot).not.toBe(first.remoteRoot); + await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: nextRun.id, lease: prepared.lease, + target: { ...target, remoteCwd: prepared.remoteRoot }, assertOwned: async () => undefined }); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, nextRun.id)); + } finally { vi.unstubAllEnvs(); } + }, 30_000); + it("keeps verified copyback files in the preserved workspace on ordinary Retry", async () => { const f = await seed(); await recoverRetainedWorkspace(f.input);