diff --git a/doc/tailscale-environments.md b/doc/tailscale-environments.md index cdcfd7d9f3b..f197a6bdb21 100644 --- a/doc/tailscale-environments.md +++ b/doc/tailscale-environments.md @@ -61,6 +61,36 @@ SSH environments display it as on; opening or upgrading an installation does not rewrite their saved configuration. The same option applies to direct SSH and Tailscale connections. +## SSH on a volume-free Cloud stack + +Cloud SSH execution uses the same durable task workspace storage as Cloud +sandboxes. Claude Code and Codex use their existing SSH adapters with +`engine=cli`. ACP archive staging cannot use an in-place Cloud workspace. An +explicit engine choice preserves the agent's execution and permission settings. +The selected +SSH account supplies the installed harness; the agent's Paperclip AI account +supplies authentication unless the remote authentication option is enabled. + +Each task receives a private directory below the configured SSH workspace path, +inside `.paperclip-workspaces`. The task runs in that remote directory. Core +keeps the workspace binding and saved file versions in its database and object +storage. A Cloud restart reconnects to the same remote files. Restoring a saved +workspace to another allocation uses the existing snapshot verification and +single-writer rules. It does not restore private provider credentials from a +task snapshot. + +The SSH machine needs Node.js and Git on its login PATH in addition to its agent +harness. The workspace owner uses Node.js for preparation and verified file +archives. SSH access alone does not qualify these dependencies or the selected +AI account. + +An existing server-local workspace is not automatically imported. Cloud still +rejects local filesystem authority and unsupported adapters. An uncertain SSH +exit or failed save retains the remote files and the writer hold; it does not +authorize another run to overwrite them. Paperclip does not stop or delete the +SSH computer. This change does not add automatic reconciliation of an unknown +remote provider process after an SSH connection is lost. + The environment keeps the `ssh` driver. Machine selection stores the connector ID and Tailscale’s stable device ID, and runtime resolution looks up its current address through that connection’s private diff --git a/packages/adapter-utils/src/ssh-command-managed.test.ts b/packages/adapter-utils/src/ssh-command-managed.test.ts index 16bc00c9f64..62ea5e09a2e 100644 --- a/packages/adapter-utils/src/ssh-command-managed.test.ts +++ b/packages/adapter-utils/src/ssh-command-managed.test.ts @@ -1,8 +1,9 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); @@ -23,14 +24,16 @@ import { createSshCommandManagedRuntimeRunner, runSshCommand } from "./ssh.js"; const roots: string[] = []; afterEach(async () => { + vi.useRealTimers(); vi.unstubAllEnvs(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function runner() { +async function runner(fileSync = false) { const cwd = await mkdtemp(path.join(os.tmpdir(), "ssh-managed-argv-")); roots.push(cwd); return createSshCommandManagedRuntimeRunner({ + fileSync, spec: { host: "fixture.invalid", username: "fixture", port: 22, remoteCwd: cwd, remoteWorkspacePath: cwd, @@ -40,6 +43,59 @@ async function runner() { } describe("SSH command-managed argv", () => { + it("opts into bounded streaming archive transfer without changing ordinary SSH runners", async () => { + const plain = await runner(); + expect(plain.syncIn).toBeUndefined(); + expect(plain.syncOut).toBeUndefined(); + const syncing = await runner(true); + const directory = await mkdtemp(path.join(os.tmpdir(), "ssh-archive-transfer-")); + roots.push(directory); + const source = path.join(directory, "source.pcws"), remote = path.join(directory, "quote's remote.pcws"), received = path.join(directory, "received.pcws"); + const contents = Buffer.alloc(20 * 1024 * 1024, 0xa5); + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + await writeFile(source, contents); + await expect(syncing.syncIn!([{ operationId: "upload", files: [{ kind: "file", sourcePath: source, targetPath: remote }] }])) + .resolves.toMatchObject({ operations: [{ filesTransferred: 1, bytesTransferred: contents.length }] }); + await expect(syncing.syncOut!([{ operationId: "download", files: [{ kind: "file", sourcePath: remote, targetPath: received }] }])) + .resolves.toMatchObject({ operations: [{ filesTransferred: 1, bytesTransferred: contents.length }] }); + expect(digest(await readFile(received))).toBe(digest(contents)); + expect((await readdir(directory)).some(name => name.includes(".upload-"))).toBe(false); + await expect(syncing.syncOut!([{ operationId: "overwrite", files: [{ kind: "file", sourcePath: remote, targetPath: received }] }])) + .rejects.toThrow(); + expect(digest(await readFile(received))).toBe(digest(contents)); + }); + + it("refuses unsupported directory and post-upload operations before transfer", async () => { + const syncing = await runner(true); + const files = [{ kind: "directory" as const, sourcePath: "/source", targetPath: "/target" }]; + const calls = vi.mocked(spawn).mock.calls.length; + await expect(syncing.syncIn!([{ operationId: "directory", files }])).rejects.toThrow("archive files only"); + expect(vi.mocked(spawn).mock.calls).toHaveLength(calls); + await expect(syncing.syncOut!([], { minimumTimeoutMs: Infinity })).rejects.toThrow("Invalid SSH transfer timeout"); + }); + + it("terminates a stalled archive upload at its bounded deadline", async () => { + const syncing = await runner(true); + const source = path.join(roots.at(-1)!, "source.pcws"); + await writeFile(source, "archive fixture"); + const actual = await vi.importActual("node:child_process"); + let started!: () => void; + const ready = new Promise(resolve => { started = resolve; }); + let child: ReturnType | undefined; + vi.mocked(spawn).mockImplementationOnce((_file, _args, options) => { + child = actual.spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], options); + started(); + return child; + }); + vi.useFakeTimers(); + const transfer = syncing.syncIn!([{ operationId: "stall", files: [{ kind: "file", sourcePath: source, targetPath: source + ".remote" }] }]); + const rejected = expect(transfer).rejects.toThrow("SSH archive upload timed out"); + await ready; + await vi.advanceTimersByTimeAsync(300_000); + await rejected; + expect(child?.killed).toBe(true); + }); + it.each(["spawn", "command"])("decodes with BSD-compatible flags and preserves stdin through %s", async mode => { const bin = await mkdtemp(path.join(os.tmpdir(), "ssh-bsd-base64-")); roots.push(bin); diff --git a/packages/adapter-utils/src/ssh.ts b/packages/adapter-utils/src/ssh.ts index 7943cd2438a..ebcd7295d45 100644 --- a/packages/adapter-utils/src/ssh.ts +++ b/packages/adapter-utils/src/ssh.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { pipeline, Transform } from "node:stream"; import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js"; +import { MAX_SANDBOX_SYNC_TIMEOUT_MS, type SandboxSyncOperation, type SandboxSyncResult } from "./sandbox-managed-runtime.js"; import { createUnrelatedHistoryGraftCommit, GIT_SYNC_COMMIT_IDENTITY_ARGS, @@ -54,6 +55,7 @@ export function createSshCommandManagedRuntimeRunner(input: { spec: SshRemoteExecutionSpec; defaultCwd?: string | null; maxBufferBytes?: number | null; + fileSync?: boolean; }): CommandManagedRuntimeRunner { const defaultCwd = input.defaultCwd?.trim() || input.spec.remoteCwd; const maxBufferBytes = @@ -62,6 +64,10 @@ export function createSshCommandManagedRuntimeRunner(input: { : 1024 * 1024; return { + ...(input.fileSync ? { + syncIn: (operations, options) => transferSshWorkspaceFiles(input.spec, "in", operations, options?.minimumTimeoutMs), + syncOut: (operations, options) => transferSshWorkspaceFiles(input.spec, "out", operations, options?.minimumTimeoutMs), + } satisfies Pick : {}), execute: async (commandInput): Promise => { const startedAt = new Date().toISOString(); const command = commandInput.command.trim(); @@ -125,6 +131,37 @@ export function createSshCommandManagedRuntimeRunner(input: { }; } +async function transferSshWorkspaceFiles(spec: SshRemoteExecutionSpec, direction: "in" | "out", + operations: SandboxSyncOperation[], minimumTimeoutMs = 300_000): Promise { + if (!Number.isFinite(minimumTimeoutMs) || minimumTimeoutMs <= 0) throw new Error("Invalid SSH transfer timeout"); + const timeoutMs = Math.min(MAX_SANDBOX_SYNC_TIMEOUT_MS, Math.max(300_000, minimumTimeoutMs)); + for (const operation of operations) { + if (operation.postUploadCommands?.length || operation.files.some(file => file.kind !== "file")) { + throw new Error("SSH workspace transport supports archive files only"); + } + } + const results: SandboxSyncResult["operations"] = []; + for (const operation of operations) { + let bytesTransferred = 0; + for (const file of operation.files) { + if (direction === "in") { + const source = await fs.lstat(file.sourcePath); + if (!source.isFile()) throw new Error("SSH archive source must be a regular file"); + const temporary = `${file.targetPath}.upload-${randomUUID()}`; + await streamLocalFileToSsh({ spec, localFile: file.sourcePath, timeoutMs, + remoteScript: `umask 077; trap ${shellQuote(`rm -f ${shellQuote(temporary)}`)} EXIT; cat > ${shellQuote(temporary)} && mv -f ${shellQuote(temporary)} ${shellQuote(file.targetPath)}` }); + bytesTransferred += source.size; + } else { + await streamSshToLocalFile({ spec, localFile: file.targetPath, timeoutMs, exclusive: true, + remoteScript: `test -f ${shellQuote(file.sourcePath)} && test ! -L ${shellQuote(file.sourcePath)} && cat ${shellQuote(file.sourcePath)}` }); + bytesTransferred += (await fs.stat(file.targetPath)).size; + } + } + results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred }); + } + return { operations: results }; +} + export interface SshEnvLabSupport { supported: boolean; reason: string | null; @@ -679,6 +716,7 @@ async function streamLocalFileToSsh(input: { localFile: string; remoteScript: string; progress?: TransferProgress; + timeoutMs?: number; }): Promise { const auth = await createSshAuthArgs(input.spec); const sshArgs = [ @@ -689,6 +727,7 @@ async function streamLocalFileToSsh(input: { `sh -c ${shellQuote(input.remoteScript)}`, ]; + let timer: ReturnType | undefined; await new Promise((resolve, reject) => { const source = createReadStream(input.localFile); const ssh = spawn("ssh", sshArgs, { @@ -706,6 +745,8 @@ async function streamLocalFileToSsh(input: { reject(error); }; + if (input.timeoutMs) timer = setTimeout(() => fail(new Error("SSH archive upload timed out")), input.timeoutMs); + ssh.stderr?.on("data", (chunk) => { sshStderr += String(chunk); }); @@ -726,7 +767,7 @@ async function streamLocalFileToSsh(input: { } resolve(); }); - }).finally(auth.cleanup); + }).finally(async () => { clearTimeout(timer); await auth.cleanup(); }); } async function streamSshToLocalFile(input: { @@ -734,6 +775,8 @@ async function streamSshToLocalFile(input: { remoteScript: string; localFile: string; progress?: TransferProgress; + timeoutMs?: number; + exclusive?: boolean; }): Promise { const auth = await createSshAuthArgs(input.spec); const sshArgs = [ @@ -744,11 +787,12 @@ async function streamSshToLocalFile(input: { `sh -c ${shellQuote(input.remoteScript)}`, ]; + let timer: ReturnType | undefined; await new Promise((resolve, reject) => { const ssh = spawn("ssh", sshArgs, { stdio: ["ignore", "pipe", "pipe"], }); - const sink = createWriteStream(input.localFile, { mode: 0o600 }); + const sink = createWriteStream(input.localFile, { mode: 0o600, ...(input.exclusive ? { flags: "wx" } : {}) }); let sshStderr = ""; let settled = false; @@ -771,6 +815,8 @@ async function streamSshToLocalFile(input: { reject(error); }; + if (input.timeoutMs) timer = setTimeout(() => fail(new Error("SSH archive download timed out")), input.timeoutMs); + const onStreamFinished = (error: NodeJS.ErrnoException | null) => { if (error) { fail(error); @@ -796,7 +842,7 @@ async function streamSshToLocalFile(input: { sshClosed = true; settle(); }); - }).finally(auth.cleanup); + }).finally(async () => { clearTimeout(timer); await auth.cleanup(); }); } async function importGitWorkspaceToSsh(input: { diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 75f8532993b..e41359333c6 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -294,8 +294,8 @@ describe("environmentRunOrchestrator — realizeForRun", () => { expect(runtime.acquireRunLease).toHaveBeenCalledWith(expect.objectContaining({ assertCompanyBinding: true })); }); - it("realizes an enrolled remote workspace in place and persists its authoritative remote root", async () => { - const environment = { ...makeEnvironment("sandbox"), id: randomUUID() }; + it.each(["sandbox", "ssh"] as const)("realizes an enrolled %s workspace in place and persists its authoritative remote root", async driver => { + const environment = { ...makeEnvironment(driver), id: randomUUID() }; const lease = makeLease({ id: randomUUID(), environmentId: environment.id, provider: "daytona", providerLeaseId: "sandbox-allocation" }); const workspace = makePersistedExecutionWorkspace({ id: randomUUID(), providerType: "cloud_sandbox", metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, source: { kind: "empty" }, @@ -303,7 +303,9 @@ describe("environmentRunOrchestrator — realizeForRun", () => { } }); const remoteRoot = `/sandbox/.paperclip-workspaces/${workspace.id}`; const savedLease = { ...lease, metadata: { remoteCwd: remoteRoot } }; - const target = { kind: "remote", transport: "sandbox", remoteCwd: "/sandbox", runner: { execute: vi.fn() } }; + const target = driver === "ssh" + ? { kind: "remote", transport: "ssh", remoteCwd: "/sandbox", spec: { remoteCwd: "/sandbox" } } + : { kind: "remote", transport: "sandbox", remoteCwd: "/sandbox", runner: { execute: vi.fn() } }; mockResolveEnvironmentExecutionTarget.mockResolvedValue(target); mockGetStoredWorkspace.mockResolvedValue({ ...workspace, cwd: remoteRoot }); mockUpdateLeaseMetadata.mockResolvedValue(savedLease); @@ -312,7 +314,7 @@ describe("environmentRunOrchestrator — realizeForRun", () => { } as never); try { const runtime = makeMockRuntime(); - const input = { ...makeRealizeInput({ environment, lease, persistedExecutionWorkspace: workspace }), adapterType: "paperclip_runner" }; + const input = { ...makeRealizeInput({ environment, lease, persistedExecutionWorkspace: workspace }), adapterType: driver === "ssh" ? "claude_local" : "paperclip_runner" }; const result = await environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }).realizeForRun(input); expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ workspace, target, source: { kind: "empty" } })); expect(mockBuildWorkspaceRealizationRequest).not.toHaveBeenCalled(); @@ -320,6 +322,9 @@ describe("environmentRunOrchestrator — realizeForRun", () => { expect(result.executionTarget).toMatchObject({ remoteCwd: remoteRoot, workspaceRealization: { mode: "in_place", authoritativeRoot: remoteRoot, outboundRestorePaths: [] } }); expect(input.executionWorkspace.cwd).toBe(remoteRoot); + if (result.executionTarget?.kind === "remote" && result.executionTarget.transport === "ssh") { + expect(result.executionTarget.spec.remoteCwd).toBe(remoteRoot); + } expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(lease.id, expect.objectContaining({ remoteCwd: remoteRoot })); expect(result.lease.metadata?.remoteCwd).toBe(remoteRoot); } finally { prepare.mockRestore(); } diff --git a/server/src/__tests__/ephemeral-server-contract.test.ts b/server/src/__tests__/ephemeral-server-contract.test.ts index e677036a7e2..a98cec86604 100644 --- a/server/src/__tests__/ephemeral-server-contract.test.ts +++ b/server/src/__tests__/ephemeral-server-contract.test.ts @@ -1,6 +1,6 @@ import { afterEach, expect, it, vi } from "vitest"; import { assertEphemeralServerConfig } from "../services/ephemeral-server-contract.js"; -import { assertEphemeralWorkspaceExecution, assertEphemeralNativeRuntime } from "../services/remote-workspaces.js"; +import { assertEphemeralWorkspaceExecution, assertEphemeralNativeRuntime, shouldUseRemoteWorkspaces } from "../services/remote-workspaces.js"; import { assertEphemeralAgentInstructions } from "../services/agent-instructions.js"; const config = { databaseUrl: "postgresql://runtime:fixture@ep-example-pooler.neon.test/tenant", storageProvider: "s3" as const }; @@ -41,3 +41,18 @@ it("requires remote workspace authority and native execution independently of pr expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false })).not.toThrow(); expect(() => assertEphemeralNativeRuntime(false)).not.toThrow(); }); + +it.each(["claude_local", "codex_local"])("admits %s over SSH only with durable remote workspace authority", adapterType => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + vi.stubEnv("DATABASE_URL", config.databaseUrl); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", env.PAPERCLIP_SECRETS_MASTER_KEY); + const remote = { externalConversation: false, adapterType, environmentDriver: "ssh", pluginTaskWorkspace: false, remoteAuthority: true }; + expect(shouldUseRemoteWorkspaces({ adapterType, environmentDriver: "ssh", experimentalEnabled: true })).toBe(true); + expect(() => assertEphemeralWorkspaceExecution(remote)).not.toThrow(); + expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false })).toThrow(); + expect(() => assertEphemeralWorkspaceExecution({ ...remote, environmentDriver: "local" })).toThrow(); + expect(() => assertEphemeralWorkspaceExecution({ ...remote, adapterType: "process" })).toThrow(); + expect(() => assertEphemeralNativeRuntime(false, { ...remote, remoteCwd: "/workspace", authorityCwd: "/workspace" })).not.toThrow(); + expect(() => assertEphemeralNativeRuntime(false, { ...remote, remoteCwd: "/other", authorityCwd: "/workspace" })).toThrow(); +}); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 941af0cbb5f..75a28cc58ef 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -1,6 +1,6 @@ import { RetainedSandboxAbsentError } from "./retained-sandbox-absent-error.js"; import { freshWorkspaceSchema } from "@paperclipai/plugin-sdk"; -import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRepositoryPath } from "./remote-workspaces.js"; +import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRepositoryPath, supportsSshRemoteWorkspace } from "./remote-workspaces.js"; /** * Centralized environment run orchestrator. * @@ -559,15 +559,15 @@ export function environmentRunOrchestrator( }; const remoteAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata); if (remoteAuthority && persistedExecutionWorkspace) { - if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") { - throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a qualified native sandbox"); + if (!(adapterType === "paperclip_runner" && environment.driver === "sandbox") && !supportsSshRemoteWorkspace(adapterType, environment.driver)) { + throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a supported remote environment"); } const target = withAcquisitionReceipt(await resolveEnvironmentExecutionTarget({ db, companyId, adapterType, environment, leaseId: lease.id, leaseMetadata: lease.metadata, lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null, })); - if (!target || target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { - throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a sandbox file transport"); + if (!target || target.kind !== "remote" || (target.transport === "sandbox" && (!target.runner || target.runnerTask))) { + throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a remote file transport"); } const prepared = await prepareRemoteWorkspace({ db, workspace: persistedExecutionWorkspace, runId: heartbeatRunId, lease, target, @@ -596,6 +596,7 @@ export function environmentRunOrchestrator( input.executionWorkspace.cwd = prepared.remoteRoot; const executionTarget: AdapterExecutionTarget = { ...target, remoteCwd: prepared.remoteRoot, + ...(target.transport === "ssh" ? { spec: { ...target.spec, remoteCwd: prepared.remoteRoot } } : {}), workspaceRealization: { mode: "in_place", authoritativeRoot: prepared.remoteRoot, pathAliases: [], outboundRestorePaths: [] }, }; diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 5eb74902d72..5ff0cc20362 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1091,6 +1091,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe if (nativeWorkspaceSync || remoteWorkspaceAuthority) { const owned = await measureRunPhase("execution.workspace_ownership", () => withNativeWorkspaceFinalizationOwnership({ db, companyId: run.companyId, runId: run.id, + ...(remoteWorkspaceAuthority && nativeRuntimeResolution.kind !== "native" && persistedExecutionWorkspace + ? { remoteWorkspaceId: persistedExecutionWorkspace.id } : {}), }, completeWorkspace)); if (!owned.acquired) throw new NativeWorkspaceFinalizationBusyError(); } else { diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index 60f2cf75db7..e5a7e4ad989 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -942,6 +942,42 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled(); }); + it.each(["claude_local", "codex_local"])("keeps %s SSH runtime and its durable workspace receipt on a volume-free server", async adapterType => { + const { input, run, agent, issue } = await fixture(adapterType); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + const environmentId = randomUUID(), leaseId = randomUUID(), workspaceId = randomUUID(); + const remoteCwd = "/home/fixture/work/.paperclip-workspaces/task"; + const reference = { schema: "paperclip.remote-workspace-run.v1", workspaceId, epoch: 1, + leaseId, providerLeaseId: "ssh-fixture", remoteCwd, state: "prepared", snapshotId: null }; + const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: agent.companyId, + name: "SSH task files", mode: "isolated_workspace", strategyType: "project_primary", providerType: "cloud_sandbox", cwd: remoteCwd, + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, source: { kind: "empty" }, + writerRunId: run.id, dirty: true, snapshotId: null, location: { environmentId, leaseId, providerLeaseId: "ssh-fixture", remoteCwd } } } }).returning(); + await db.update(heartbeatRuns).set({ runnerProfileJson: { remoteWorkspace: reference } }).where(eq(heartbeatRuns.id, run.id)); + input.workspace.persistedExecutionWorkspace = workspace; + input.workspace.executionTarget = { kind: "remote", transport: "ssh", environmentId, leaseId, remoteCwd, + workspaceRealization: { mode: "in_place", authoritativeRoot: remoteCwd, pathAliases: [], outboundRestorePaths: [] }, + spec: { host: "fixture.invalid", username: "fixture", port: 22, remoteWorkspacePath: "/home/fixture/work", remoteCwd, + useHostAuth: false, privateKey: null, knownHosts: null, strictHostKeyChecking: true } }; + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, + runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: input.workspace.executionTarget, workspaceId }); + const selected = await selectHeartbeatRuntime(db, input); + expect(selected.selected).toBe(true); + if (!selected.selected) throw new Error("Expected legacy SSH runtime selection"); + expect(selected.nativeExecution).toBeNull(); + expect((await read(run.id)).runnerProfileJson?.remoteWorkspace).toEqual(reference); + expect(await coordinators(run.id)).toHaveLength(0); + expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled(); + input.workspace.executionTarget.spec.remoteCwd = "/other-directory"; + await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("native Paperclip Runner"); + input.workspace.executionTarget.spec.remoteCwd = remoteCwd; + input.workspace.persistedExecutionWorkspace.companyId = randomUUID(); + await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("native Paperclip Runner"); + input.workspace.persistedExecutionWorkspace.companyId = agent.companyId; + input.workspace.executionTarget.leaseId = randomUUID(); + await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("native Paperclip Runner"); + }); + it.each(["warm", "per_turn"] as const)("preserves resolved %s policy on an ephemeral server", async mode => { const { input, run } = await fixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 4d7896f7b74..2462b365c50 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -125,7 +125,7 @@ export interface HeartbeatRuntimeSelectionInput { getFreshSessionHandoff: (() => Promise) | undefined; }; workspace: { - persistedExecutionWorkspace: { id: string; metadata?: Record | null } | null; + persistedExecutionWorkspace: { id: string; companyId?: string; metadata?: Record | null } | null; executionWorkspace: Omit[0]["workspace"], "id">; executionTarget: Parameters[0]["target"]; activeEnvironmentLease: (Pick[0], "lease"> & { @@ -325,7 +325,18 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele const { runtimeConfig, runtimeSkillEntries, runScopedMentionedSkillKeys, instructionCopy, agentIdentity, issueAssigneeOverrides, resolvedConfig, managedAiRuntime } = input.config; const runOptions = input.recovery; const { providerTraceRequested, providerTraceCapture } = input.trace; - if (!isDotRun) assertEphemeralNativeRuntime(nativeRuntimeResolution.kind === "native"); + const sshAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata); + if (!isDotRun) assertEphemeralNativeRuntime(nativeRuntimeResolution.kind === "native", { + adapterType: agent.adapterType, + environmentDriver: executionTarget?.kind === "remote" ? executionTarget.transport : null, + remoteAuthority: Boolean(sshAuthority?.location && executionTarget?.kind === "remote" && executionTarget.transport === "ssh" + && persistedExecutionWorkspace?.companyId === agent.companyId && sshAuthority.writerRunId === run.id + && executionTarget.spec.remoteCwd === executionTarget.remoteCwd + && sshAuthority.location.environmentId === executionTarget.environmentId + && sshAuthority.location.leaseId === executionTarget.leaseId), + remoteCwd: executionTarget?.kind === "remote" ? executionTarget.remoteCwd : null, + authorityCwd: sshAuthority?.location?.remoteCwd ?? null, + }); let nativeExecution: NativeExecutionInput | null = null; let toolRefreshOnResume: boolean | undefined; let getNativeFreshSessionHandoff: (() => Promise) | undefined; @@ -1184,6 +1195,9 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele else '{}'::jsonb end) || (case when ${heartbeatRuns.runnerProfileJson} ? 'adapterDispatch' then jsonb_build_object('adapterDispatch', ${heartbeatRuns.runnerProfileJson}->'adapterDispatch') + else '{}'::jsonb end) + || (case when ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,schema}' = 'paperclip.remote-workspace-run.v1' + then jsonb_build_object('remoteWorkspace', ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace') else '{}'::jsonb end) || ${JSON.stringify(providerTraceRequested ? { providerTrace: { mode: "raw", traceId: providerTraceCapture?.metadata.id ?? null, maxBytes: PROVIDER_TRACE_MAX_BYTES } } : {})}::jsonb`, updatedAt: new Date(), }) diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index f86ac513682..220a8d06c89 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -590,6 +590,31 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace expect(await db.select().from(executionWorkspaces)).toHaveLength(0); }); + it.each(["claude_local", "codex_local"])("reserves remote storage for volume-free %s SSH tasks before acquiring the machine", async adapterType => { + const { input, issue, envOrchestrator } = await fixture(); + configureRemotePersistence(); + process.env.PAPERCLIP_EPHEMERAL_SERVER = "true"; + input.agent.adapterType = adapterType; + await db.update(agents).set({ adapterType }).where(eq(agents.id, input.agent.id)); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id)); + input.task.issueRef = { ...issue, projectId: null, projectWorkspaceId: null }; + input.task.context = { issueId: issue.id }; + input.policy.useIsolatedTaskDirectory = true; + const environment = await environmentService(db).create(input.agent.companyId, { name: `SSH ${adapterType}`, driver: "ssh", status: "active", envVars: {}, + config: { host: "fixture.invalid", port: 22, username: "fixture", remoteWorkspacePath: "/home/fixture/work", useHostAuth: false } }); + input.environment.selectedEnvironmentId = environment.id; + input.environment.selectedEnvironmentForConfig = environment; + const resolveHost = vi.spyOn(input.services, "resolveWorkspaceForRun"); + const acquire = vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async () => { throw new Error("SSH acquisition admitted"); }); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("SSH acquisition admitted"); + expect(acquire).toHaveBeenCalledOnce(); + expect(resolveHost).not.toHaveBeenCalled(); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.companyId, input.agent.companyId)); + expect(readRemoteWorkspaceAuthority(workspace.metadata)).toMatchObject({ source: { kind: "empty" }, writerRunId: input.run.id }); + const [bound] = await db.select().from(issues).where(eq(issues.id, issue.id)); + expect(bound.executionWorkspaceId).toBe(workspace.id); + }); + it.each([false, true])("configured local folders retain host authority only on durable servers (ephemeral=%s)", async ephemeral => { const { input, envOrchestrator } = await fixture(); input.agent.adapterType = "paperclip_runner"; diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index e3eb9b09da7..851da8f5724 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -285,6 +285,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace }); if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, adapterType: agent.adapterType, + environmentDriver: selectedEnvironmentForConfig?.driver, pluginTaskWorkspace, remoteAuthority }); const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({ db, companyId: agent.companyId, runId: run.id, @@ -370,7 +371,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace // Source resolution can retain a configured host folder. Check the // resolved authority before materializing it or acquiring a sandbox. if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, - adapterType: agent.adapterType, pluginTaskWorkspace, remoteAuthority }); + adapterType: agent.adapterType, environmentDriver: selectedEnvironmentForConfig?.driver, pluginTaskWorkspace, remoteAuthority }); } else { remoteSourceState.value = sourceState; diff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts index c7a1416638e..d9bb5eb8efc 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts @@ -10,8 +10,8 @@ import { nativeSha256 } from "./canonical.js"; import { NATIVE_WORKSPACE_OWNER_UNVERIFIED, NATIVE_WORKSPACE_OWNER_WAIT_NEXT_ACTION } from "./native-workspace-finalization-recovery-state.js"; import { randomUUID } from "node:crypto"; import os from "node:os"; -import { and, eq, inArray, sql } from "drizzle-orm"; -import { heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, withDedicatedDbConnection, type Db } from "@paperclipai/db"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; +import { executionWorkspaces, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { currentNativeWorkspaceRecoveryResult } from "./native-workspace-recovery-binding.js"; import { withWorkspaceLockKeepalive } from "./native-workspace-lock-keepalive.js"; import { @@ -105,7 +105,7 @@ export function nativeWorkspaceFinalizationOwnerCanUpgrade(profile: Record( - input: { db: Db; companyId: string; runId: string; retainedRepairActionId?: string }, + input: { db: Db; companyId: string; runId: string; retainedRepairActionId?: string; remoteWorkspaceId?: string }, action: (ownership: NativeWorkspaceFinalizationOwnership) => Promise, ): Promise<{ acquired: false; reason: NativeWorkspaceFinalizationWaitReason } | { acquired: true; value: T }> { const identity = await currentNativeControllerIdentity(); @@ -120,9 +120,22 @@ export async function withNativeWorkspaceFinalizationOwnership( ...(deployment ? { exclusiveVolume: deployment.exclusiveVolume } : {}), ...(deployment?.exclusiveVolume && deployment.volumeId ? { volumeId: deployment.volumeId } : {}), }; - const scope = and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), - input.retainedRepairActionId ? inArray(heartbeatRuns.runtimeMode, ["native", "legacy"]) : eq(heartbeatRuns.runtimeMode, "native")); - const owned = and(scope, sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY}->>'token' = ${owner.token}`); + const remoteLegacy = input.remoteWorkspaceId ? and(eq(heartbeatRuns.runtimeMode, "legacy"), + sql`${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,schema}' = 'paperclip.remote-workspace-run.v1'`, + sql`${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,workspaceId}' = ${input.remoteWorkspaceId}`, + sql`exists (select 1 from ${executionWorkspaces} where ${executionWorkspaces.id} = ${input.remoteWorkspaceId}::uuid + and ${executionWorkspaces.companyId} = ${heartbeatRuns.companyId} + and ${executionWorkspaces.metadata}#>>'{remoteWorkspace,schema}' = 'paperclip.remote-workspace.v1' + and (${executionWorkspaces.metadata}#>>'{remoteWorkspace,writerRunId}' = ${heartbeatRuns.id}::text + or (${executionWorkspaces.metadata}#>>'{remoteWorkspace,writerRunId}' is null + and ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,state}' = 'saved')) + and ${executionWorkspaces.metadata}#>>'{remoteWorkspace,epoch}' = ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,epoch}' + and ${executionWorkspaces.metadata}#>>'{remoteWorkspace,location,leaseId}' = ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,leaseId}')`) : undefined; + const runScope = and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), + input.retainedRepairActionId || input.remoteWorkspaceId ? inArray(heartbeatRuns.runtimeMode, ["native", "legacy"]) + : eq(heartbeatRuns.runtimeMode, "native")); + const claimScope = and(runScope, input.retainedRepairActionId ? undefined : or(eq(heartbeatRuns.runtimeMode, "native"), remoteLegacy)); + const owned = and(runScope, sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY}->>'token' = ${owner.token}`); const runKey = nativeWorkspaceFinalizationLockKey(input.companyId, input.runId); let claimed = false; let actionPromise: Promise | undefined; @@ -139,7 +152,7 @@ export async function withNativeWorkspaceFinalizationOwnership( // Reads and durable progress use the ordinary pool, not the lock transaction. const [run] = await input.db.select({ profile: heartbeatRuns.runnerProfileJson, issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId')`, - agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(scope).limit(1); + agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(claimScope).limit(1); if (!run) throw new Error("native_workspace_finalization_binding_missing"); const rawPrior = run.profile?.[OWNER_KEY]; const prior = readOwner(rawPrior); @@ -154,7 +167,7 @@ export async function withNativeWorkspaceFinalizationOwnership( const [issue] = await db.select().from(issues).where(and( eq(issues.companyId, input.companyId), eq(issues.id, run.issueId), )).for("update"); - const [currentRun] = await db.select().from(heartbeatRuns).where(scope).for("update"); + const [currentRun] = await db.select().from(heartbeatRuns).where(claimScope).for("update"); if (input.retainedRepairActionId) { const [repair] = await db.select().from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, run.issueId), @@ -276,7 +289,7 @@ export async function withNativeWorkspaceFinalizationOwnership( claimed = true; const [claimedRun] = await db.update(heartbeatRuns).set({ runnerProfileJson: sql`jsonb_set(coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb), array[${OWNER_KEY}], ${JSON.stringify(owner)}::jsonb) || ${JSON.stringify(platformProof ? { nativeWorkspaceOwnerStopProof: platformProof } : {})}::jsonb`, - }).where(and(scope, prior + }).where(and(claimScope, prior ? sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY} = ${JSON.stringify(rawPrior)}::jsonb` : sql`${heartbeatRuns.runnerProfileJson}->>${OWNER_KEY} is null` )).returning({ id: heartbeatRuns.id }); diff --git a/server/src/services/remote-workspace-recovery-attachments.ts b/server/src/services/remote-workspace-recovery-attachments.ts index 687a77b5671..1702294baf8 100644 --- a/server/src/services/remote-workspace-recovery-attachments.ts +++ b/server/src/services/remote-workspace-recovery-attachments.ts @@ -1,8 +1,7 @@ import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; -import { hasRemoteRecoveryAttachment, removeRemoteRecoveryAttachment } from "./remote-workspace-transport.js"; +import { hasRemoteRecoveryAttachment, removeRemoteRecoveryAttachment, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js"; import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { LEGACY_WORKSPACE_RECOVERY_MIGRATION } from "./legacy-workspace-recovery-migration.js"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -225,7 +224,7 @@ function record(value: unknown): Record { * provenance. This checks the selected root; it never imports or replaces files. */ export async function retainRemoteRecoveryAttachments(input: { db: Db; workspace: Pick; - runId: string; remoteRoot: string; target: AdapterSandboxExecutionTarget; assertOwned?: () => Promise; + runId: string; remoteRoot: string; target: RemoteWorkspaceTarget; assertOwned?: () => Promise; recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string; provenance: string; retired?: true }>; retiredOperationIds: ReadonlySet; storage?: StorageService; }) { diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 8e80375eecb..305149fce90 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -1,8 +1,10 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { lstat, mkdtemp, realpath, rm, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { createSshCommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/ssh"; import type { RemoteWorkspaceSource } from "@paperclipai/shared"; import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, remoteWorkspaceCaptureFailureMessage, remoteWorkspaceFailureDetail } from "./remote-workspace-snapshot-archive.js"; import { readWorkspaceInventory, readonlyWorkspaceInventoryScript } from "./remote-workspace-readonly-inventory.js"; @@ -13,7 +15,8 @@ import { classifyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/work import { MAX_SANDBOX_SYNC_TIMEOUT_MS, type SandboxSyncOperation } from "@paperclipai/adapter-utils/sandbox-managed-runtime"; import { REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH } from "./remote-workspace-archive-io.js"; -export type RemoteWorkspaceTarget = Extract; +export type RemoteWorkspaceTarget = Extract + | (Extract & { runner: CommandManagedRuntimeRunner }); async function transferArchive(target: RemoteWorkspaceTarget, direction: "syncIn" | "syncOut", operations: SandboxSyncOperation[], byteSize: number) { @@ -30,11 +33,34 @@ const checkoutConfiguredRefScript = String.raw`if(input.ref){const remoteRef=inp await git(['checkout','--detach',selected.stdout.trim()]);}`; export function requireRemoteWorkspaceTarget(target: AdapterExecutionTarget | null): RemoteWorkspaceTarget { + if (target?.kind === "remote" && target.transport === "ssh") { + return { ...target, runner: createSshCommandManagedRuntimeRunner({ spec: { ...target.spec, remoteCwd: target.remoteCwd }, fileSync: true }) }; + } if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask || !target.runner.syncIn || !target.runner.syncOut) throw new Error("remote_workspace_transport_unsupported"); return target; } +export async function syncRemoteWorkspaceRecoveryDirectory(input: { + target: RemoteWorkspaceTarget; sourcePath: string; targetPath: string; +}) { + if (input.target.transport === "sandbox") { + await input.target.runner!.syncIn!([{ operationId: randomUUID(), files: [{ + kind: "directory", sourcePath: input.sourcePath, targetPath: input.targetPath, + }] }]); + return; + } + const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-recovery-transfer-")); + try { + const archivePath = path.join(scratch, "recovery.pcws"); + const captured = await captureRemoteWorkspaceDirectory({ cwd: input.sourcePath, archivePath }); + const restorationIdentity = createHash("sha256").update(JSON.stringify([input.targetPath, captured.manifestDigest])).digest("hex"); + const restored = await restoreRemoteWorkspaceDirectory({ target: input.target, remoteCwd: input.targetPath, + archivePath, manifestDigest: captured.manifestDigest, restorationIdentity }); + await cleanupRemoteWorkspaceArchive({ target: input.target, remoteCwd: restored.stagingRoot, operationId: restored.operationId }); + } finally { await removeTaskDirectoryTree(scratch); } +} + /** The workspace archive is built inside the sandbox, so a full sandbox disk stops the save. */ export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL_CODE = "remote_workspace_sandbox_disk_full"; export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index c3d3d8f29dd..41ef78e92f3 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -1802,7 +1802,7 @@ require('node:module').syncBuiltinESMExports();`); it("preserves enrolled workspaces after disabling enrollment and rejects a legacy writer", async () => { const reserved = await reserve({ kind: "empty" }); expect(shouldUseRemoteWorkspaces({ adapterType: "paperclip_runner", environmentDriver: "sandbox", experimentalEnabled: false, existingWorkspace: reserved })).toBe(true); - expect(() => shouldUseRemoteWorkspaces({ adapterType: "codex_local", environmentDriver: "sandbox", experimentalEnabled: false, existingWorkspace: reserved })).toThrow("native sandbox"); + expect(() => shouldUseRemoteWorkspaces({ adapterType: "codex_local", environmentDriver: "sandbox", experimentalEnabled: false, existingWorkspace: reserved })).toThrow("supported remote execution environment"); }); it("keeps enrollment conditional while enrolled authority fails closed when persistence disappears", async () => { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 14b8939aada..23e9f3c6bbc 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -33,7 +33,7 @@ import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, remoteWorkspaceFailureCause, hasRemoteRecoveryAttachment, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, RemoteWorkspaceRestoreError, type RemoteWorkspaceRestoreStep, - type RemoteWorkspaceSavePhaseTiming, + type RemoteWorkspaceSavePhaseTiming, syncRemoteWorkspaceRecoveryDirectory, } from "./remote-workspace-transport.js"; import { beginWorkspaceSnapshot, findUnchangedWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, newWorkspaceSnapshotStorageTiming, uploadWorkspaceSnapshot, @@ -94,25 +94,32 @@ function sameExportLocation(value: unknown, location: NonNullable recorded[key] === location[key]); } +export function supportsSshRemoteWorkspace(adapterType: string, environmentDriver: string | null | undefined) { + return environmentDriver === "ssh" && ["claude_local", "codex_local", "paperclip_runner"].includes(adapterType); +} + export function shouldUseRemoteWorkspaces(input: { adapterType: string; environmentDriver: string | null; experimentalEnabled: boolean; existingWorkspace?: { metadata?: Record | null } | null; }) { const enrolled = readRemoteWorkspaceAuthority(input.existingWorkspace?.metadata) !== null; - if (enrolled && (input.adapterType !== "paperclip_runner" || input.environmentDriver !== "sandbox")) { - throw conflict("This workspace requires its native sandbox execution environment.", { code: "remote_workspace_execution_required" }); + const supported = input.adapterType === "paperclip_runner" && input.environmentDriver === "sandbox" + || supportsSshRemoteWorkspace(input.adapterType, input.environmentDriver); + if (enrolled && !supported) { + throw conflict("This workspace requires a supported remote execution environment.", { code: "remote_workspace_execution_required" }); } if (enrolled) return true; // Existing host authority changes only through the explicit import workflow. - if (input.existingWorkspace || !input.experimentalEnabled || input.adapterType !== "paperclip_runner" - || input.environmentDriver !== "sandbox") return false; + if (input.existingWorkspace || !input.experimentalEnabled || !supported) return false; return configuredRemoteWorkspacePersistence() !== null; } export function assertEphemeralWorkspaceExecution(input: { externalConversation: boolean; adapterType: string; pluginTaskWorkspace: boolean; remoteAuthority: boolean; + environmentDriver?: string | null; }) { if (process.env.PAPERCLIP_EPHEMERAL_SERVER !== "true" || input.externalConversation) return; + if (input.remoteAuthority && supportsSshRemoteWorkspace(input.adapterType, input.environmentDriver)) return; // Runtime selection has not promoted a fresh run yet. Validate storage here; if (input.adapterType !== "paperclip_runner" || (!input.pluginTaskWorkspace && !input.remoteAuthority)) { @@ -121,8 +128,13 @@ export function assertEphemeralWorkspaceExecution(input: { } } -export function assertEphemeralNativeRuntime(native: boolean) { - if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && !native) { +export function assertEphemeralNativeRuntime(native: boolean, ssh?: { + adapterType: string; environmentDriver: string | null; remoteAuthority: boolean; + remoteCwd: string | null; authorityCwd: string | null; +}) { + const durableSsh = ssh?.remoteAuthority && supportsSshRemoteWorkspace(ssh.adapterType, ssh.environmentDriver) + && Boolean(ssh.remoteCwd) && ssh.remoteCwd === ssh.authorityCwd; + if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && !native && !durableSsh) { throw unprocessable("Volume-free servers require native Paperclip Runner execution with remote workspace storage.", { code: "ephemeral_execution_configuration_required" }); } @@ -556,9 +568,8 @@ export async function prepareRemoteWorkspace(input: { const provenance = await readFile(path.join(item.path, "provenance.json"), "utf8"); const exists = await hasRemoteRecoveryAttachment({ target, remoteCwd: remoteRoot, operationId: item.operationId, provenance }); - if (!exists) await target.runner!.syncIn!([{ operationId: randomUUID(), files: [{ - kind: "directory", sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative), - }] }]); + if (!exists) await syncRemoteWorkspaceRecoveryDirectory({ target, + sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative) }); recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: relative, provenance }); } } finally { await removeTaskDirectoryTree(scratch); } diff --git a/server/src/services/ssh-remote-workspace.integration.test.ts b/server/src/services/ssh-remote-workspace.integration.test.ts new file mode 100644 index 00000000000..3e820846f2c --- /dev/null +++ b/server/src/services/ssh-remote-workspace.integration.test.ts @@ -0,0 +1,274 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, authUsers, companyMemberships, closeRegisteredClients, companies, createDb, environmentLeases, environments, + executionWorkspaces, heartbeatRuns, issues } from "@paperclipai/db"; +import type { AdapterSshExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { buildSshEnvLabFixtureConfig, getSshEnvLabSupport, startSshEnvLabFixture, + stopSshEnvLabFixture, type SshEnvLabFixtureState } from "@paperclipai/adapter-utils/ssh"; +import type { EnvironmentLease } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { createStorageService } from "../storage/service.js"; +import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, reserveRemoteWorkspace, saveRemoteWorkspace } from "./remote-workspaces.js"; +import { requireRemoteWorkspaceTarget } from "./remote-workspace-transport.js"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { withNativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js"; +import { heartbeatService } from "./heartbeat.js"; +import { aiConnectionService } from "./ai-connections.js"; +import * as storageModule from "../storage/index.js"; +import { collectEnvironmentSecretRefs, normalizeEnvironmentConfigForPersistence } from "./environment-config.js"; +import { secretService } from "./secrets.js"; +import * as runLogModule from "./run-log-store.js"; +import { objectLogFixture } from "./heartbeat/__tests__/object-log-fixture.js"; +import { issueService } from "./issues.js"; +import { getServerAdapter } from "../adapters/index.js"; +import * as recoverySnapshots from "./workspace-recovery-snapshot.js"; +import { hasRemoteRecoveryAttachment } from "./remote-workspace-transport.js"; + +const [postgres, ssh] = await Promise.all([getEmbeddedPostgresTestSupport(), getSshEnvLabSupport()]); + +describe.skipIf(!postgres.supported || !ssh.supported)("Cloud SSH durable workspace with a real SSH server", () => { + let root: string; + let database: Awaited>; + let db: ReturnType; + let sshd: SshEnvLabFixtureState | undefined; + + beforeAll(async () => { + root = await realpath(await mkdtemp(path.join(os.tmpdir(), "cloud-ssh-workspace-"))); + database = await startEmbeddedPostgresTestDatabase("cloud-ssh-workspace-"); + db = createDb(database.connectionString); + vi.stubEnv("DATABASE_URL", database.connectionString); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "controller-home")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "62".repeat(32)); + sshd = await startSshEnvLabFixture({ statePath: path.join(root, "sshd", "state.json") }); + }, 60_000); + + afterAll(async () => { + if (sshd) await stopSshEnvLabFixture(sshd); + if (database) await closeRegisteredClients(database.connectionString); + await database?.cleanup(); + if (root) await rm(root, { recursive: true, force: true }); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }, 60_000); + + it("keeps Hello World through controller replacement and restores it to a new SSH allocation", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "SSH workspace test", issuePrefix: "SSH" }); + await db.insert(agents).values({ id: agentId, companyId, name: "SSH Claude", adapterType: "claude_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Write Hello World", assigneeAgentId: agentId }); + const config = await buildSshEnvLabFixtureConfig(sshd!); + const [environment] = await db.insert(environments).values({ driver: "ssh", name: "SSH test", config: {} }).returning(); + const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))), provider: "s3" as const }; + + async function acquire(id: string, remoteCwd = config.remoteWorkspacePath, providerLeaseId = "ssh-fixture") { + await db.insert(heartbeatRuns).values({ id, companyId, agentId, issueId, runtimeMode: "legacy", status: "running" }); + const [lease] = await db.insert(environmentLeases).values({ companyId, environmentId: environment.id, + heartbeatRunId: id, issueId, provider: "ssh", providerLeaseId, status: "active", + metadata: { driver: "ssh", remoteCwd } }).returning(); + const target: AdapterSshExecutionTarget = { kind: "remote", transport: "ssh", environmentId: environment.id, + leaseId: lease.id, remoteCwd, spec: { ...config, remoteCwd, useHostAuth: false } }; + return { lease: lease as EnvironmentLease, target }; + } + + async function stop(id: string, lease: EnvironmentLease) { + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, id)); + const [stopped] = await db.update(environmentLeases).set({ metadata: { ...lease.metadata, + remoteExecutionTermination: remoteTerminationReceipt(lease, { state: "stopped", providerLeaseId: lease.providerLeaseId }) } }) + .where(eq(environmentLeases.id, lease.id)).returning(); + return stopped as EnvironmentLease; + } + + const first = await acquire(runId); + const reserved = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: null, source: { kind: "empty" }, mode: "shared_workspace" }); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId, ...first }); + const firstTarget = { ...first.target, remoteCwd: prepared.remoteRoot, spec: { ...first.target.spec, remoteCwd: prepared.remoteRoot } }; + const result = await requireRemoteWorkspaceTarget(firstTarget).runner!.execute({ command: "node", args: ["-e", + "require('fs').writeFileSync('hello_world.txt','Hello World\\n')"], cwd: prepared.remoteRoot, timeoutMs: 10_000 }); + expect(result.exitCode).toBe(0); + const stopped = await stop(runId, prepared.lease); + await expect(withNativeWorkspaceFinalizationOwnership({ db, companyId, runId }, async () => {})) + .rejects.toThrow("binding_missing"); + await expect(withNativeWorkspaceFinalizationOwnership({ db, companyId, runId, remoteWorkspaceId: randomUUID() }, async () => {})) + .rejects.toThrow("binding_missing"); + await expect(withNativeWorkspaceFinalizationOwnership({ db, companyId, runId, remoteWorkspaceId: prepared.workspace.id }, ownership => + saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, lease: stopped, + target: { ...firstTarget, spec: { ...firstTarget.spec, host: "127.0.0.1", port: 1 } }, assertOwned: ownership.assertHeld }))) + .rejects.toThrow(); + const [held] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, reserved.id)); + expect(readRemoteWorkspaceAuthority(held.metadata)).toMatchObject({ writerRunId: runId, dirty: true, + snapshotId: readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.snapshotId }); + expect(await readFile(path.join(prepared.remoteRoot, "hello_world.txt"), "utf8")).toBe("Hello World\n"); + const blockedId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: blockedId, companyId, agentId, issueId, runtimeMode: "legacy", status: "running" }); + await expect(reserveRemoteWorkspace({ db, storage, companyId, issueId, runId: blockedId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, + existingWorkspaceId: prepared.workspace.id, source: { kind: "empty" } })).rejects.toMatchObject({ status: 409 }); + const peerIssueId = randomUUID(), peerRunId = randomUUID(); + await db.insert(issues).values({ id: peerIssueId, companyId, title: "Use the saved shared workspace", assigneeAgentId: agentId, + executionWorkspaceId: prepared.workspace.id }); + await db.insert(heartbeatRuns).values({ id: peerRunId, companyId, agentId, issueId: peerIssueId, runtimeMode: "legacy", status: "running" }); + const finalized = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId, remoteWorkspaceId: prepared.workspace.id }, async ownership => { + await saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, + lease: stopped, target: firstTarget, assertOwned: ownership.assertHeld }); + const handedOn = await reserveRemoteWorkspace({ db, storage, companyId, issueId: peerIssueId, runId: peerRunId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: prepared.workspace.id, source: { kind: "empty" } }); + expect(readRemoteWorkspaceAuthority(handedOn.metadata)).toMatchObject({ epoch: 2, writerRunId: peerRunId }); + await ownership.assertHeld(); + await expect(saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, + lease: stopped, target: firstTarget, assertOwned: ownership.assertHeld })).rejects.toThrow("authority_changed"); + }); + expect(finalized.acquired).toBe(true); + const [finishedOwner] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(finishedOwner.runnerProfileJson).not.toHaveProperty("nativeWorkspaceFinalizationOwner"); + await expect(withNativeWorkspaceFinalizationOwnership({ db, companyId, runId, remoteWorkspaceId: prepared.workspace.id }, async () => {})) + .rejects.toThrow("binding_missing"); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, peerRunId)); + + // Replace the controller connection. The saved task and remote files are the authority. + db = createDb(database.connectionString); + const [saved] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, reserved.id), eq(executionWorkspaces.companyId, companyId))); + const snapshotId = readRemoteWorkspaceAuthority(saved.metadata)!.snapshotId; + expect(snapshotId).toBeTruthy(); + expect(await readFile(path.join(prepared.remoteRoot, "hello_world.txt"), "utf8")).toBe("Hello World\n"); + + const followupId = randomUUID(); + const followup = await acquire(followupId); + const reused = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId: followupId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: saved.id, source: { kind: "empty" } }); + const continued = await prepareRemoteWorkspace({ db, storage, workspace: reused, runId: followupId, ...followup }); + expect(continued.remoteRoot).toBe(prepared.remoteRoot); + await saveRemoteWorkspace({ db, storage, workspace: continued.workspace, runId: followupId, + lease: await stop(followupId, continued.lease), target: followup.target, assertOwned: async () => {} }); + + const [latest] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, saved.id)); + const restoredBase = path.join(root, "replacement-remote"); + await mkdir(restoredBase); + const restoreId = randomUUID(), replacement = await acquire(restoreId, restoredBase, "replacement-ssh-allocation"); + const restoring = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId: restoreId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: latest.id, source: { kind: "empty" } }); + const restored = await prepareRemoteWorkspace({ db, storage, workspace: restoring, runId: restoreId, ...replacement }); + expect(restored.remoteRoot).not.toBe(prepared.remoteRoot); + expect(await readFile(path.join(restored.remoteRoot, "hello_world.txt"), "utf8")).toBe("Hello World\n"); + }, 60_000); + + it("prepares a verified recovered copy on SSH without directory-sync support", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), sourceId = randomUUID(), operationId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "SSH recovery test", issuePrefix: "REC" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Recovery worker", adapterType: "claude_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Recover earlier files", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, runtimeMode: "legacy", status: "running" }); + const [environment] = await db.insert(environments).values({ driver: "ssh", name: "SSH recovery", config: {} }).returning(); + const config = await buildSshEnvLabFixtureConfig(sshd!); + const [lease] = await db.insert(environmentLeases).values({ companyId, environmentId: environment.id, + heartbeatRunId: runId, issueId, provider: "ssh", providerLeaseId: "ssh-recovery", status: "active" }).returning(); + const target: AdapterSshExecutionTarget = { kind: "remote", transport: "ssh", environmentId: environment.id, + leaseId: lease.id, remoteCwd: config.remoteWorkspacePath, spec: { ...config, remoteCwd: config.remoteWorkspacePath } }; + const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, "recovery-objects"))), provider: "s3" as const }; + const reserved = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: null, + source: { kind: "empty" }, recoverySourceRunIds: [sourceId] }); + const provenance = JSON.stringify({ operationId, sourceRunId: sourceId }); + // Start at the verified extraction boundary; source authorization and snapshot verification keep their existing owner. + const extract = vi.spyOn(recoverySnapshots, "attachPreservedWorkspaceSnapshots").mockImplementationOnce(async input => { + expect(input.sourceRunIds).toContain(sourceId); + const recovered = path.join(input.cwd, "recovered", operationId); + await mkdir(path.join(recovered, "files", "nested"), { recursive: true }); + await writeFile(path.join(recovered, "files", "nested", "preserved.txt"), "Earlier work\n"); + await writeFile(path.join(recovered, "provenance.json"), provenance); + return [{ operationId, sourceRunId: sourceId, path: recovered }]; + }); + try { + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId, lease: lease as EnvironmentLease, target }); + const recovered = path.join(prepared.remoteRoot, ".paperclip-recovered", operationId); + expect(await readFile(path.join(recovered, "files", "nested", "preserved.txt"), "utf8")).toBe("Earlier work\n"); + expect(await readFile(path.join(recovered, "provenance.json"), "utf8")).toBe(provenance); + expect(await hasRemoteRecoveryAttachment({ target: requireRemoteWorkspaceTarget(target), remoteCwd: prepared.remoteRoot, operationId, provenance })).toBe(true); + await writeFile(path.join(recovered, "files", "nested", "preserved.txt"), "A later edit\n"); + await prepareRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId, lease: prepared.lease, target }); + expect(await readFile(path.join(recovered, "files", "nested", "preserved.txt"), "utf8")).toBe("A later edit\n"); + } finally { extract.mockRestore(); } + }, 60_000); + + it("runs an assigned Claude task through heartbeat with the configured Paperclip account", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_AGENT_JWT_SECRET", "synthetic-cloud-ssh-jwt-secret"); + vi.stubEnv("AWS_EC2_METADATA_DISABLED", "true"); + const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, "heartbeat-objects"))), provider: "s3" as const }; + vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage); + vi.spyOn(runLogModule, "getRunLogStore").mockReturnValue(objectLogFixture()); + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), userId = `ssh-user-${randomUUID()}`; + await db.insert(authUsers).values({ id: userId, name: "SSH test user", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Cloud SSH task", issuePrefix: "HSSH", defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" }); + const account = await aiConnectionService(db).save(companyId, userId, { provider: "anthropic", method: "api_key", + ownership: "personal", name: "Selected test account", apiKey: "synthetic-selected-ssh-key", agentIds: [], allAgents: true }, "synthetic-selected-ssh-key"); + const config = await buildSshEnvLabFixtureConfig(sshd!); + const command = path.join(root, "fake-claude"); + await writeFile(command, `#!${process.execPath} +if (process.argv.includes('--version')) { console.log('2.1.0'); process.exit(0); } +if (process.env.ANTHROPIC_API_KEY !== 'synthetic-selected-ssh-key') process.exit(91); +const fs = require('fs'), path = require('path'); +const history = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture-history'); +const resumed = process.argv.includes('--resume'); +if (resumed && !fs.existsSync(history)) process.exit(92); +fs.mkdirSync(path.dirname(history), {recursive:true}); fs.writeFileSync(history, 'provider conversation state'); +fs.writeFileSync('continuation.json', JSON.stringify({resumed})); +require('fs').writeFileSync('hello_world.txt', 'Hello World\\n'); +console.log(JSON.stringify({type:'result',subtype:'success',is_error:false,result:'Created hello_world.txt',session_id:'12345678-1234-4abc-9def-123456789012',total_cost_usd:0,usage:{input_tokens:1,output_tokens:1}})); +`, { mode: 0o700 }); + const storedConfig = await normalizeEnvironmentConfigForPersistence({ db, companyId, environmentName: "Heartbeat SSH", + driver: "ssh", secretProvider: "local_encrypted", config: { ...config, useHostAuth: false } }); + const [environment] = await db.insert(environments).values({ driver: "ssh", name: "Heartbeat SSH", config: storedConfig }).returning(); + await secretService(db).replaceSecretRefsForInstanceTarget({ targetType: "environment", targetId: environment.id }, + await collectEnvironmentSecretRefs({ db, environment: { id: environment.id, driver: "ssh", config: storedConfig } })); + await db.insert(agents).values({ id: agentId, companyId, name: "Claude SSH task", adapterType: "claude_local", status: "idle", + defaultEnvironmentId: environment.id, adapterConfig: { command, engine: "cli", model: "claude-sonnet-4-6", timeoutSec: 20 }, + runtimeConfig: { aiConnection: { provider: "anthropic", method: "api_key", mode: "delegated", connectionId: account.connectionId, grantId: account.grantId } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Create hello_world.txt", status: "todo", assigneeAgentId: agentId, + responsibleUserId: userId, issueNumber: 1 }); + const adapter = getServerAdapter("claude_local"), execute = adapter.execute; + expect(adapter.type).toBe("claude_local"); + vi.spyOn(adapter, "execute").mockImplementation(async input => { + const result = await execute(input); + // The synthetic CLI stands in for the provider's final task update too. + if (result.exitCode === 0) await db.update(issues).set({ status: "in_review" }).where(eq(issues.id, issueId)); + return result; + }); + const heartbeat = heartbeatService(db); + try { + const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "manual", + payload: { issueId }, contextSnapshot: { issueId } }); + expect(run).toBeTruthy(); + await heartbeat.drainActiveRunExecutions(); + const finished = await heartbeat.getRun(run!.id); + if (finished?.status !== "succeeded") throw new Error(JSON.stringify({ errorCode: finished?.errorCode, + error: finished?.error, log: await heartbeat.readLog(run!.id, { limitBytes: 12_000 }).catch(() => null) })); + expect(finished).toMatchObject({ status: "succeeded", runtimeMode: "legacy" }); + const authority = finished!.runnerProfileJson!.remoteWorkspace as { workspaceId: string; state: string; remoteCwd: string }; + expect(authority.state).toBe("saved"); + expect(await readFile(path.join(authority.remoteCwd, "hello_world.txt"), "utf8")).toBe("Hello World\n"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, authority.workspaceId)); + expect(readRemoteWorkspaceAuthority(workspace.metadata)).toMatchObject({ writerRunId: null, dirty: false }); + const comment = await issueService(db).addComment(issueId, "Confirm the file still exists.", { userId }); + const followup = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "manual", + payload: { issueId }, contextSnapshot: { issueId, wakeCommentId: comment.id } }); + await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + await heartbeat.drainActiveRunExecutions(); + const continued = await heartbeat.getRun(followup!.id); + if (continued?.status !== "succeeded") throw new Error(JSON.stringify({ status: continued?.status, + retryReason: continued?.scheduledRetryReason, retryAt: continued?.scheduledRetryAt, errorCode: continued?.errorCode, + error: continued?.error, result: continued?.resultJson, log: await heartbeat.readLog(followup!.id, { limitBytes: 12_000 }).catch(() => null) })); + expect(continued).toMatchObject({ status: "succeeded" }); + expect(JSON.parse(await readFile(path.join(authority.remoteCwd, "continuation.json"), "utf8"))).toEqual({ resumed: true }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 60_000); +});