diff --git a/packages/adapter-utils/src/execution-target.test.ts b/packages/adapter-utils/src/execution-target.test.ts index 28f239e0f9d..1449d11d904 100644 --- a/packages/adapter-utils/src/execution-target.test.ts +++ b/packages/adapter-utils/src/execution-target.test.ts @@ -207,6 +207,20 @@ describe("runAdapterExecutionTargetShellCommand", () => { expect(onLog).toHaveBeenCalledWith("stderr", "partial stderr"); }); + it("returns a timedOut result for the rejection runSshCommand gives when its timeout kills ssh", async () => { + vi.spyOn(ssh, "runSshCommand").mockRejectedValue(Object.assign(new Error("Process exited with code -1"), { + code: null, + signal: "SIGTERM", + killed: true, + stdout: "", + stderr: "", + })); + + const result = await runAdapterExecutionTargetShellCommand("run-2b", sshExecutionTarget(), "sleep 10", { cwd: "/tmp/local", env: {} }); + + expect(result).toMatchObject({ exitCode: null, signal: "SIGTERM", timedOut: true }); + }); + it("returns the SSH process exit code for non-zero remote command failures", async () => { vi.spyOn(ssh, "runSshCommand").mockRejectedValue(Object.assign(new Error("non-zero exit"), { code: 17, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 077cb38b63d..70b2f652c00 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1035,10 +1035,12 @@ export async function runAdapterExecutionTargetShellCommand( stdout?: string; stderr?: string; signal?: string | null; + killed?: boolean; }; const stdout = timedOutError.stdout ?? ""; const stderr = timedOutError.stderr ?? ""; - if (typeof timedOutError.code === "number") { + const timedOut = timedOutError.killed === true || timedOutError.code === "ETIMEDOUT"; + if (!timedOut && typeof timedOutError.code === "number") { if (stdout) await onLog("stdout", stdout); if (stderr) await onLog("stderr", stderr); return { @@ -1051,7 +1053,7 @@ export async function runAdapterExecutionTargetShellCommand( startedAt, }; } - if (timedOutError.code !== "ETIMEDOUT") { + if (!timedOut) { throw error; } if (stdout) await onLog("stdout", stdout); diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 31cbd5b5823..85a7d24242f 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -936,7 +936,8 @@ describe("persistent agent directories", () => { const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValue({ exitCode: 1, signal: null, timedOut: false, stdout: "", stderr: "fixture failure", pid: null, startedAt: null }); try { await expect(copies.prepare({ ...target(), runId, cwd: home, target: { kind: "remote", transport: "ssh", environmentId: randomUUID(), remoteCwd: "/fixture/task", - spec: { host: "unused.invalid", port: 22, username: "test", remoteCwd: "/fixture/task", remoteWorkspacePath: "/fixture/task", privateKey: null, knownHosts: null, strictHostKeyChecking: true } } })).rejects.toThrow("Could not exclude"); + spec: { host: "unused.invalid", port: 22, username: "test", remoteCwd: "/fixture/task", remoteWorkspacePath: "/fixture/task", privateKey: null, knownHosts: null, strictHostKeyChecking: true } } })).rejects.toThrow("Could not exclude agent files from task Git staging (reason=exit_1)"); + expect(shell.mock.calls.filter(([, , command]) => command.includes("Unsafe runtime exclusion"))).toHaveLength(1); const row = (await copies.get(companyId, runId))!; expect(row.state).toBe("unavailable"); expect(row.receipt?.baseline).toBeUndefined(); @@ -1475,6 +1476,43 @@ describe("persistent agent directories", () => { } finally { stage.mockRestore(); restore.mockRestore(); exclude.mockRestore(); } }); + describe("Git exclusion on a slow remote", () => { + const timedOut = { exitCode: null, signal: null, timedOut: true, stdout: "", stderr: "", pid: null, startedAt: null }; + const succeeded = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null }; + const sshTarget = (remoteCwd: string) => ({ kind: "remote" as const, transport: "ssh" as const, environmentId: randomUUID(), remoteCwd, + spec: { host: "unused.invalid", port: 22, username: "test", remoteCwd, remoteWorkspacePath: remoteCwd, privateKey: null, knownHosts: null, strictHostKeyChecking: true } }); + const exclusions = (shell: { mock: { calls: Parameters[] } }) => + shell.mock.calls.filter(([, , command]) => command.includes("Unsafe runtime exclusion")); + + it("retries a timed-out exclusion and stages the agent files", async () => { + const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValueOnce(timedOut).mockResolvedValue(succeeded); + const stage = vi.spyOn(ssh, "syncDirectoryToSsh").mockImplementation(async input => { + await fs.mkdir(path.dirname(input.remoteDir), { recursive: true }); + await fs.cp(input.localDir, input.remoteDir, { recursive: true }); + }); + try { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId }); + const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target: sshTarget(path.join(home, "slow-task")) }))!; + expect(copy.state).toBe("prepared"); + expect(exclusions(shell).map(([, , , options]) => options.timeoutSec)).toEqual([30, 30]); + expect(stage).toHaveBeenCalledWith(expect.objectContaining({ remoteDir: copy.executionRoot })); + } finally { stage.mockRestore(); shell.mockRestore(); } + }); + + it("names the timeout after the last attempt", async () => { + const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValue(timedOut); + try { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId }); + await expect(copies.prepare({ companyId, agentId, runId, cwd: home, target: sshTarget("/fixture/slow-task") })) + .rejects.toThrow("Could not exclude agent files from task Git staging (reason=timed_out)"); + expect(exclusions(shell)).toHaveLength(3); + expect((await copies.get(companyId, runId))!.state).toBe("unavailable"); + } finally { shell.mockRestore(); } + }); + }); + it("deduplicates concurrent stopped callbacks and discards completed operational snapshots", async () => { const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "note.txt"), "one write"); diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index ec4298d8035..5a51e7b7dfb 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -9,7 +9,7 @@ import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCom import { withDirectoryMergeLock, directorySnapshotSha256, parseDirectorySnapshot, serializeDirectorySnapshot, type DirectoryMergeLockOperation } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { AGENT_FILES_CONTRACT, AgentFileLimitError, agentFileStore, agentStorageWarning, inspectAgentDirectory } from "./agent-file-store.js"; import { agentInstructionsBundleMode, deriveBundleState, resolveManagedInstructionsRoot } from "./agent-instructions.js"; -import { instructionGitExcludeFailureReason, instructionGitExcludeProgram } from "./agent-instruction-files.js"; +import { excludeRuntimeFromRemoteGit } from "./agent-instruction-files.js"; import { resolveInstructionActor } from "./agent-instruction-authorization.js"; import { HttpError, conflict, notFound } from "../errors.js"; import type { AuthorizationActor } from "./authorization.js"; @@ -235,13 +235,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } try { if (input.target?.kind === "remote") { - const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; - const excluded = await runAdapterExecutionTargetShellCommand(input.runId, input.target, - `node -e ${quote(instructionGitExcludeProgram)} ${quote(input.target.remoteCwd)}`, - { cwd: input.target.remoteCwd, env: {}, timeoutSec: 15 }); - if (excluded.exitCode !== 0 || excluded.timedOut) { - throw new Error(`Could not exclude agent files from task Git staging (reason=${instructionGitExcludeFailureReason(excluded)})`); - } + await excludeRuntimeFromRemoteGit(input.runId, input.target, "Could not exclude agent files from task Git staging"); const runtime = await transport(row, input.target, false); transports.set(key(row), runtime); if (input.target.transport === "sandbox") { diff --git a/server/src/services/agent-instruction-files.ts b/server/src/services/agent-instruction-files.ts index 27d4b6704fb..fa81c24c957 100644 --- a/server/src/services/agent-instruction-files.ts +++ b/server/src/services/agent-instruction-files.ts @@ -3,6 +3,7 @@ import { constants } from "node:fs"; import path from "node:path"; import { randomUUID } from "node:crypto"; import { unprocessable } from "../errors.js"; +import { runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; export const MAX_INSTRUCTION_BYTES = 1024 * 1024; export function instructionPath(value: string): string { @@ -190,3 +191,17 @@ export function instructionGitExcludeFailureReason(result: { if (result.signal && /^SIG[A-Z0-9]{1,10}$/.test(result.signal)) return `signal_${result.signal.toLowerCase()}`; return Number.isSafeInteger(result.exitCode) ? `exit_${result.exitCode}` : "unknown"; } + +const REMOTE_GIT_EXCLUDE_ATTEMPTS = 3; + +export async function excludeRuntimeFromRemoteGit(runId: string, target: Exclude, failure: string) { + const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; + for (let attempt = 1; ; attempt++) { + const result = await runAdapterExecutionTargetShellCommand(runId, target, + `node -e ${quote(instructionGitExcludeProgram)} ${quote(target.remoteCwd)}`, { cwd: target.remoteCwd, env: {}, timeoutSec: 30 }); + if (result.exitCode === 0 && !result.timedOut) return; + if (!result.timedOut || attempt === REMOTE_GIT_EXCLUDE_ATTEMPTS) { + throw new Error(`${failure} (reason=${instructionGitExcludeFailureReason(result)})`); + } + } +} diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index b2ab372e9b2..ceb84893f8a 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -22,7 +22,7 @@ import { conflict, notFound } from "../errors.js"; import { agentInstructionsService, agentInstructionsBundleMode } from "./agent-instructions.js"; import { agentInstructionRevisionService } from "./agent-instruction-revisions.js"; import { resolveInstructionActor } from "./agent-instruction-authorization.js"; -import { assertInstructionPathSafe, instructionBytes, instructionPath, materializeInstructionBytes, MAX_INSTRUCTION_BYTES, readInstructionBytes, instructionGitExcludeProgram } from "./agent-instruction-files.js"; +import { assertInstructionPathSafe, instructionBytes, instructionPath, materializeInstructionBytes, MAX_INSTRUCTION_BYTES, readInstructionBytes, instructionGitExcludeProgram, excludeRuntimeFromRemoteGit } from "./agent-instruction-files.js"; import { hasNativeLocalProcessStop } from "./native-local-process-stop.js"; import { remoteExecutionHasStopped } from "./remote-execution-termination.js"; import type { AuthorizationActor } from "./authorization.js"; @@ -248,9 +248,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen assets: [{ key: "instructions", localDir: localRoot, followSymlinks: false }], }); if (staged.assetDirs.instructions !== executionRoot) throw new Error("Instruction copy staging path changed"); - const excluded = await runAdapterExecutionTargetShellCommand(input.runId, target, - `node -e ${quote(instructionGitExcludeProgram)} ${quote(target.remoteCwd)}`, { cwd: target.remoteCwd, env: {}, timeoutSec: 15 }); - if (excluded.exitCode !== 0 || excluded.timedOut) throw new Error("Could not exclude private instructions from Git staging"); + await excludeRuntimeFromRemoteGit(input.runId, target, "Could not exclude private instructions from Git staging"); } // A refresh does not supersede the completed turn until every local and // remote staging step succeeds. Keep its receipt and stop evidence intact