fix(ssh): support all Cloud SSH harnesses in remote task workspaces (#22078)

Support durable in-place SSH task workspaces across all declared SSH-capable
Cloud harnesses. Keep controller filesystem operations off remote paths,
preserve SSH session identity, and stage provider assets on their owning
execution target. Keep managed ACP conversation identity stable across
credential-home rotation and preserve Pi JSONL history after credential cleanup.

Verify with the real SSH heartbeat matrix, adapter and session regression
checks, and typechecking. Final CI passed on 3b629da with 57 successful checks
and three skips. Greptile scored the change 5/5 with no unresolved comments.
Live paid-provider acceptance remains unqualified.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-11-17 16:06:25 -08:00
1 parent a3f86e5cc8
commit 9032f55d83
35 files changed
+579 -102

No files matched your search

@@ -42,7 +42,7 @@ it("requires remote workspace authority and native execution independently of pr
expect(() => assertEphemeralNativeRuntime(false)).not.toThrow();
});
it.each(["claude_local", "codex_local"])("admits %s over SSH only with durable remote workspace authority", adapterType => {
it.each(["claude_local", "codex_local", "cursor", "gemini_local", "grok_local", "kimi_local", "opencode_local", "pi_local"])("admits %s over SSH only with durable remote workspace authority", adapterType => {
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
vi.stubEnv("DATABASE_URL", config.databaseUrl);
vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
@@ -0,0 +1,22 @@
import { describe, expect, it } from "vitest";
import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetSessionMatches, type AdapterSshExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { getServerAdapter } from "../adapters/index.js";
describe("SSH adapter saved session identity", () => {
it.each(["claude_local", "codex_local", "cursor", "gemini_local", "grok_local", "kimi_local", "opencode_local", "pi_local"])("keeps %s connection guards through persistence", adapterType => {
const target: AdapterSshExecutionTarget = { kind: "remote", transport: "ssh", remoteCwd: "/remote/task",
spec: { host: "ssh.example.test", port: 22, username: "fixture", remoteCwd: "/remote/task", remoteWorkspacePath: "/remote",
useHostAuth: false, privateKey: null, knownHosts: null, strictHostKeyChecking: true } };
const identity = adapterExecutionTargetSessionIdentity(target)!;
const codec = getServerAdapter(adapterType).sessionCodec!;
const stored = codec.serialize({ sessionId: "session-fixture", cwd: target.remoteCwd,
remoteExecution: { ...identity, privateKey: "credential-canary", tailscaleSocket: "/private/socket" } });
const restored = codec.deserialize(JSON.parse(JSON.stringify(stored)))!;
expect(restored.remoteExecution).toEqual(identity);
expect(adapterExecutionTargetSessionMatches(restored.remoteExecution, target)).toBe(true);
expect(adapterExecutionTargetSessionMatches(restored.remoteExecution, { ...target, spec: { ...target.spec, host: "other.example.test" } })).toBe(false);
const legacy = codec.deserialize({ sessionId: "session-fixture", cwd: target.remoteCwd })!;
expect(legacy.remoteExecution).toBeUndefined();
expect(adapterExecutionTargetSessionMatches(legacy.remoteExecution, target)).toBe(false);
});
});
@@ -942,7 +942,7 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled();
});
it.each(["claude_local", "codex_local"])("keeps %s SSH runtime and its durable workspace receipt on a volume-free server", async adapterType => {
it.each(["claude_local", "codex_local", "cursor", "gemini_local", "grok_local", "kimi_local", "opencode_local", "pi_local"])("keeps %s SSH runtime and its durable workspace receipt on a volume-free server", async adapterType => {
const { input, run, agent, issue } = await fixture(adapterType);
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
const environmentId = randomUUID(), leaseId = randomUUID(), workspaceId = randomUUID();
@@ -590,7 +590,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
});
it.each(["claude_local", "codex_local"])("reserves remote storage for volume-free %s SSH tasks before acquiring the machine", async adapterType => {
it.each(["claude_local", "codex_local", "cursor", "gemini_local", "grok_local", "kimi_local", "opencode_local", "pi_local"])("reserves remote storage for volume-free %s SSH tasks before acquiring the machine", async adapterType => {
const { input, issue, envOrchestrator } = await fixture();
configureRemotePersistence();
process.env.PAPERCLIP_EPHEMERAL_SERVER = "true";
+2 -2
View File
@@ -14,7 +14,7 @@ import {
companies, environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db,
} from "@paperclipai/db";
import {
readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema,
adapterSupportsRemoteManagedEnvironments, readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema,
type RemoteWorkspaceAuthority, type RemoteWorkspaceRunReference, type RemoteWorkspaceSource,
type EnvironmentLease,
} from "@paperclipai/shared";
@@ -95,7 +95,7 @@ function sameExportLocation(value: unknown, location: NonNullable<RemoteWorkspac
}
export function supportsSshRemoteWorkspace(adapterType: string, environmentDriver: string | null | undefined) {
return environmentDriver === "ssh" && ["claude_local", "codex_local", "paperclip_runner"].includes(adapterType);
return environmentDriver === "ssh" && adapterSupportsRemoteManagedEnvironments(adapterType);
}
export function shouldUseRemoteWorkspaces(input: {
@@ -1,3 +1,4 @@
import fs from "node:fs/promises";
import { randomUUID } from "node:crypto";
import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
import os from "node:os";
@@ -9,7 +10,7 @@ import { agents, authUsers, companyMemberships, closeRegisteredClients, companie
import type { AdapterSshExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { buildSshEnvLabFixtureConfig, getSshEnvLabSupport, shellQuote, startSshEnvLabFixture,
stopSshEnvLabFixture, type SshEnvLabFixtureState } from "@paperclipai/adapter-utils/ssh";
import type { EnvironmentLease } from "@paperclipai/shared";
import type { AiProvider, EnvironmentLease } from "@paperclipai/shared";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
@@ -195,30 +196,53 @@ describe.skipIf(!postgres.supported || !ssh.supported)("Cloud SSH durable worksp
} finally { extract.mockRestore(); }
}, 60_000);
it.each(["cli", "acp"])("runs an assigned Claude %s task through heartbeat with the configured Paperclip account", async engine => {
const harnesses = [
{ adapterType: "claude_local", provider: "anthropic", authEnv: "ANTHROPIC_API_KEY", model: "claude-sonnet-4-6", engines: ["cli", "acp"] },
{ adapterType: "codex_local", provider: "openai", authEnv: "OPENAI_API_KEY", model: "gpt-5.4-mini", engines: ["cli", "acp"] },
{ adapterType: "gemini_local", provider: "google", authEnv: "GEMINI_API_KEY", model: "gemini-2.5-pro", engines: ["cli", "acp"] },
{ adapterType: "grok_local", provider: "xai", authEnv: "XAI_API_KEY", model: "grok-4", engines: ["cli"] },
{ adapterType: "opencode_local", provider: "openrouter", authEnv: "OPENROUTER_API_KEY", model: "openrouter/openai/gpt-4.1", engines: ["cli"] },
{ adapterType: "pi_local", provider: "openai", authEnv: "OPENAI_API_KEY", model: "openai/gpt-5.4-mini", engines: ["cli"] },
{ adapterType: "cursor", provider: null, authEnv: "PAPERCLIP_SSH_FIXTURE_AUTH", model: "auto", engines: ["cli"] },
{ adapterType: "kimi_local", provider: null, authEnv: "PAPERCLIP_SSH_FIXTURE_AUTH", model: "kimi-k2.5", engines: ["cli"] },
];
const runs = harnesses.flatMap(harness => harness.engines.map(engine => ({ ...harness, engine, key: `${harness.adapterType}-${engine}` })));
it.each(runs)("runs an assigned $key task through heartbeat over SSH", async ({ adapterType, provider, authEnv, model, engine, key }) => {
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
vi.stubEnv("PAPERCLIP_AGENT_JWT_SECRET", "synthetic-cloud-ssh-jwt-secret");
vi.stubEnv("AWS_EC2_METADATA_DISABLED", "true");
const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, `heartbeat-${engine}-objects`))), provider: "s3" as const };
const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, `heartbeat-${key}-objects`))), provider: "s3" as const };
vi.spyOn(storageModule, "getStorageService").mockReturnValue(storage);
vi.spyOn(runLogModule, "getRunLogStore").mockReturnValue(objectLogFixture());
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), userId = `ssh-user-${randomUUID()}`;
await db.insert(authUsers).values({ id: userId, name: "SSH test user", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companies).values({ id: companyId, name: "Cloud SSH task", issuePrefix: engine === "cli" ? "HCLI" : "HACP", defaultResponsibleUserId: userId });
await db.insert(companies).values({ id: companyId, name: "Cloud SSH task", issuePrefix: `SSH${runs.findIndex(run => run.key === key)}`, defaultResponsibleUserId: userId });
await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" });
const account = await aiConnectionService(db).save(companyId, userId, { provider: "anthropic", method: "api_key",
ownership: "personal", name: "Selected test account", apiKey: "synthetic-selected-ssh-key", agentIds: [], allAgents: true }, "synthetic-selected-ssh-key");
const account = provider ? await aiConnectionService(db).save(companyId, userId, { provider: provider as AiProvider, method: "api_key",
ownership: "personal", name: "Selected test account", apiKey: "synthetic-selected-ssh-key", agentIds: [], allAgents: true }, "synthetic-selected-ssh-key") : null;
const config = await buildSshEnvLabFixtureConfig(sshd!);
const command = path.join(root, `fake-claude-${engine}`);
const command = path.join(root, `fake-${key}`);
await writeFile(command, `#!${process.execPath}
if (process.argv.includes('--version')) { console.log('2.1.0'); process.exit(0); }
if (process.env.ANTHROPIC_API_KEY !== 'synthetic-selected-ssh-key') process.exit(91);
if (process.argv.includes('models')) { console.log(${JSON.stringify(model)}); process.exit(0); }
if (process.env[${JSON.stringify(authEnv)}] !== 'synthetic-selected-ssh-key') process.exit(91);
const fs = require('fs'), path = require('path');
const history = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture-history');
const harness = ${JSON.stringify(adapterType)};
const piSession = harness === 'pi_local' ? process.argv[process.argv.indexOf('--session') + 1] : null;
const history = harness === 'claude_local' ? path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture-history')
: piSession || path.join(process.cwd(), '.fixture-provider-history');
const runTask = (resumed) => {
if (resumed && !fs.existsSync(history)) throw Error('Missing provider conversation history');
fs.mkdirSync(path.dirname(history), {recursive:true}); fs.writeFileSync(history, 'provider conversation state');
fs.mkdirSync(path.dirname(history), {recursive:true});
if (piSession) {
const previous = resumed ? fs.readFileSync(piSession,'utf8') : '';
if (resumed && JSON.parse(previous.split('\\n')[0]).cwd !== process.cwd()) throw Error('Invalid Pi session header');
if (resumed && !previous.includes('first-turn-history')) throw Error('Missing Pi conversation messages');
if (!resumed) fs.writeFileSync(piSession,JSON.stringify({type:'session',version:3,id:'fixture',cwd:process.cwd()})+'\\n');
fs.appendFileSync(piSession,JSON.stringify({type:'message',message:{role:'assistant',content:resumed?'continued-history':'first-turn-history'}})+'\\n');
fs.writeFileSync('pi-session-path.json',JSON.stringify({session:piSession,home:process.env.HOME}));
} else fs.writeFileSync(history, 'provider conversation state');
fs.writeFileSync('continuation.json', JSON.stringify({resumed}));
require('fs').writeFileSync('hello_world.txt', 'Hello World\\n');
};
@@ -246,26 +270,63 @@ if (process.argv.includes('--acp-fixture')) {
} catch (error) { send({jsonrpc:'2.0',id:request.id,error:{code:-32603,message:error.message}}); }
});
} else {
runTask(process.argv.includes('--resume'));
console.log(JSON.stringify({type:'result',subtype:'success',is_error:false,result:'Created hello_world.txt',session_id:'12345678-1234-4abc-9def-123456789012',total_cost_usd:0,usage:{input_tokens:1,output_tokens:1}}));
const id = '12345678-1234-4abc-9def-123456789012';
const resumed = ['--resume', 'resume', '-r'].some(arg => process.argv.includes(arg))
|| (harness === 'opencode_local' && process.argv.includes('--session'))
|| Boolean(piSession && fs.existsSync(piSession) && fs.statSync(piSession).size > 0);
runTask(resumed);
const emit = value => console.log(JSON.stringify(value));
if (harness === 'codex_local') {
emit({type:'thread.started',thread_id:id}); emit({type:'item.completed',item:{type:'agent_message',text:'Created hello_world.txt'}});
emit({type:'turn.completed',usage:{input_tokens:1,output_tokens:1}});
} else if (harness === 'gemini_local') {
emit({type:'system',subtype:'init',session_id:id}); emit({type:'message',role:'assistant',content:'Created hello_world.txt'});
emit({type:'result',status:'success',session_id:id,stats:{input_tokens:1,output_tokens:1}});
} else if (harness === 'kimi_local') {
emit({role:'assistant',content:'Created hello_world.txt'}); emit({role:'meta',type:'session.resume_hint',session_id:id});
} else if (harness === 'opencode_local') {
emit({type:'text',sessionID:id,part:{text:'Created hello_world.txt'}});
emit({type:'step_finish',sessionID:id,part:{cost:0,tokens:{input:1,output:1,cache:{read:0,write:0}}}});
} else if (harness === 'pi_local') {
emit({type:'turn_end',message:{role:'assistant',content:'Created hello_world.txt',usage:{input:1,output:1}},toolResults:[]});
} else if (harness === 'grok_local') {
emit({type:'message',data:'Created hello_world.txt'}); emit({type:'end',sessionId:id,stopReason:'EndTurn',usage:{input_tokens:1,output_tokens:1}});
} else emit({type:'result',subtype:'success',is_error:false,result:'Created hello_world.txt',session_id:id,total_cost_usd:0,usage:{input_tokens:1,output_tokens:1}});
}
`, { mode: 0o700 });
const storedConfig = await normalizeEnvironmentConfigForPersistence({ db, companyId, environmentName: `Heartbeat SSH ${engine}`,
driver: "ssh", secretProvider: "local_encrypted", config: { ...config, useHostAuth: false } });
const [environment] = await db.insert(environments).values({ driver: "ssh", name: `Heartbeat SSH ${engine}`, config: storedConfig }).returning();
const storedConfig = await normalizeEnvironmentConfigForPersistence({ db, companyId, environmentName: `Heartbeat SSH ${key}`,
driver: "ssh", secretProvider: "local_encrypted", config: { ...config, useHostAuth: !provider } });
const [environment] = await db.insert(environments).values({ driver: "ssh", name: `Heartbeat SSH ${key}`, config: storedConfig }).returning();
await secretService(db).replaceSecretRefsForInstanceTarget({ targetType: "environment", targetId: environment.id },
await collectEnvironmentSecretRefs({ db, environment: { id: environment.id, driver: "ssh", config: storedConfig } }));
const adapterConfig = { command, ...(engine === "cli" ? { engine: "cli" } : { agentCommand: `${shellQuote(command)} --acp-fixture`, acpWarmHandleIdleMs: 60_000 }),
model: "claude-sonnet-4-6", timeoutSec: 20 };
await db.insert(agents).values({ id: agentId, companyId, name: "Claude SSH task", adapterType: "claude_local", status: "idle",
model, timeoutSec: 20, ...(!provider ? { env: { [authEnv]: "synthetic-selected-ssh-key" } } : {}) };
await db.insert(agents).values({ id: agentId, companyId, name: `${adapterType} SSH task`, adapterType, status: "idle",
defaultEnvironmentId: environment.id, adapterConfig,
runtimeConfig: { aiConnection: { provider: "anthropic", method: "api_key", mode: "delegated", connectionId: account.connectionId, grantId: account.grantId } } });
runtimeConfig: account ? { aiConnection: { provider, method: "api_key", mode: "delegated", connectionId: account.connectionId, grantId: account.grantId } } : {} });
await db.insert(issues).values({ id: issueId, companyId, title: "Create hello_world.txt", status: "todo", assigneeAgentId: agentId,
responsibleUserId: userId, issueNumber: 1 });
const adapter = getServerAdapter("claude_local"), execute = adapter.execute;
expect(adapter.type).toBe("claude_local");
const adapter = getServerAdapter(adapterType), execute = adapter.execute;
expect(adapter.type).toBe(adapterType);
const executing = vi.spyOn(adapter, "execute").mockImplementation(async input => {
const result = await execute(input);
const target = input.executionTarget;
expect(target).toMatchObject({ kind: "remote", transport: "ssh", workspaceRealization: { mode: "in_place" } });
const remoteCwd = target!.kind === "remote" ? target!.remoteCwd : "";
const localAccesses: string[] = [];
const guards = (["stat", "mkdir"] as const).map(method => {
const original = fs[method].bind(fs);
return vi.spyOn(fs, method).mockImplementation(((candidate: unknown, ...args: unknown[]) => {
if (typeof candidate === "string" && (candidate === remoteCwd || candidate.startsWith(remoteCwd + path.sep))) {
localAccesses.push(method);
return Promise.reject(Object.assign(new Error("SSH workspace is unavailable on the controller"), { code: "EACCES" }));
}
return (original as (...args: unknown[]) => unknown)(candidate, ...args);
}) as never);
});
let result;
try { result = await execute(input); }
finally { for (const guard of guards) guard.mockRestore(); }
expect(localAccesses).toEqual([]);
// The synthetic provider stands in for the final task update too.
if (result.exitCode === 0) await db.update(issues).set({ status: "in_review" }).where(eq(issues.id, issueId));
return result;
@@ -283,6 +344,11 @@ if (process.argv.includes('--acp-fixture')) {
const authority = finished!.runnerProfileJson!.remoteWorkspace as { workspaceId: string; state: string; remoteCwd: string };
expect(authority.state).toBe("saved");
expect(await readFile(path.join(authority.remoteCwd, "hello_world.txt"), "utf8")).toBe("Hello World\n");
const piState = adapterType === "pi_local" ? JSON.parse(await readFile(path.join(authority.remoteCwd, "pi-session-path.json"), "utf8")) : null;
if (piState) {
expect(await readFile(piState.session, "utf8")).toContain("first-turn-history");
await expect(fs.stat(piState.home)).rejects.toMatchObject({ code: "ENOENT" });
}
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, authority.workspaceId));
expect(readRemoteWorkspaceAuthority(workspace.metadata)).toMatchObject({ writerRunId: null, dirty: false });
const comment = await issueService(db).addComment(issueId, "Confirm the file still exists.", { userId });
@@ -297,6 +363,13 @@ if (process.argv.includes('--acp-fixture')) {
expect(continued).toMatchObject({ status: "succeeded" });
const continuation = JSON.parse(await readFile(path.join(authority.remoteCwd, "continuation.json"), "utf8"));
expect(continuation).toEqual({ resumed: true });
if (piState) {
const next = JSON.parse(await readFile(path.join(authority.remoteCwd, "pi-session-path.json"), "utf8"));
expect(next.session).toBe(piState.session);
expect(next.home).not.toBe(piState.home);
expect(await readFile(next.session, "utf8")).toContain("continued-history");
await expect(fs.stat(next.home)).rejects.toMatchObject({ code: "ENOENT" });
}
} finally {
await heartbeat.drainActiveRunExecutions();
executing.mockRestore();