fix: Wait for the SSH export pipeline to drain before importing the bundle (#20708)

> [!NOTE]
> Opened by Prospector's automated issue-fix pipeline. A maintainer
reviews and decides; nothing here merges on its own.

## Thinking Path

- Issue #16567 reports a defect in paperclipai/paperclip.
- Root cause, as the fixing agent read it: SSH process closure ended the
destination stream while the progress Transform still held buffered
bytes. Transfer success now requires both pipeline completion and SSH
completion.
- This pull request: Wait for the SSH export pipeline to drain before
importing the bundle

## What Changed

- `packages/adapter-utils/src/ssh.ts`: Use Node's pipeline to drain the
stream chain, propagate stream errors, and release streams on failure
while preserving SSH stderr errors.
- `packages/adapter-utils/src/ssh-export-stream.test.ts`: Reproduced
truncation and confirmed the fix passes with and without progress. Added
failure cleanup cases; their final run and typecheck have not returned,
and live SSH fixtures skipped because ssh is unavailable.

## Verification

- Reproducing test(s), added in this PR:
`packages/adapter-utils/src/ssh-export-stream.test.ts`,
`packages/adapter-utils/src/ssh-export.test.ts`
- They fail on the base commit and pass with the fix, each twice in a
fresh sandbox.
- 3 independent attempts wrote 3 reproductions; this fix passes every
one of them.
- The repository's compile command passes with the fix applied.
- 10 existing test file(s) near the change failed, and fail on the base
too, so they cannot show whether the change breaks them.
- The full test suite passes except for the 17 file(s) that already fail
on the base commit.

## Risks

- Written by an automated pipeline, not a person: review it as you would
an outside contribution.
- Risk tier of the touched paths: 2.

## Model Used

- codex:gpt-6.1-sol, opus, sonnet (Anthropic Claude, via Claude Code;
OpenAI, via the Codex CLI), writing and checking the change in an
isolated sandbox.
- opus (Anthropic Claude, via Claude Code), reviewing it independently.

## Linked Issues

Fixes #16567

#16567 also mentions, without asking this fix for it:
- Add git bundle verify or a remote size check to provide a clearer
diagnostic for short files. (The report calls this 'A cheap extra guard
would be git bundle verify (or a size check sent by the remote script)
before git fetch'; it is an optional additional safeguard. Empty or
malformed output that ends normally still reaches the existing git fetch
validation at packages/adapter-utils/src/ssh.)

Other pull requests that name #16567:
- #16613 (open): fix: prevent truncated SSH Git exports

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [ ] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with 'Fixes: #' / 'Closes
#' / 'Refs #' OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub '#NNN' / 'github.com/paperclipai/paperclip'
URLs)
- [ ] My branch name describes the change (e.g. 'docs/...', 'fix/...')
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [ ] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [ ] I will address all Greptile and reviewer comments before
requesting merge

<!-- prospector:issue-fix v1 issue=16567 base=5ab98b006359
report=6118c07135df7c3a -->

Co-authored-by: commitperclip[bot] <284812584+commitperclip[bot]@users.noreply.github.com>
This commit is contained in:
commitperclip[bot] authored and GitHub committed 2026-11-13 12:59:04 -06:00
1 parent ed671f57bf
commit 80f7989f4b
3 files changed
+239 -14

No files matched your search

@@ -0,0 +1,103 @@
import { EventEmitter } from "node:events";
import { execFile, spawn } from "node:child_process";
import { createWriteStream } from "node:fs";
import { PassThrough, Readable, Writable } from "node:stream";
import { afterEach, expect, it, vi } from "vitest";
import { restoreWorkspaceFromSshExecution } from "./ssh.js";
vi.mock("node:child_process", async (importOriginal) => ({
...await importOriginal<typeof import("node:child_process")>(),
spawn: vi.fn(),
execFile: vi.fn(),
}));
vi.mock("node:fs", async (importOriginal) => ({
...await importOriginal<typeof import("node:fs")>(),
createWriteStream: vi.fn(),
}));
afterEach(() => vi.resetAllMocks());
it.each(["stdout", "sink", "exit", "spawn"])("rejects an SSH export %s failure and releases the streams", async (failure) => {
const error = new Error("transfer failed");
const stdout = new PassThrough();
const stderr = new PassThrough();
const sink = new Writable({ write(_chunk, _encoding, callback) { callback(); } });
const child = Object.assign(new EventEmitter(), { stdout, stderr, kill: vi.fn() });
vi.mocked(createWriteStream).mockReturnValue(sink as ReturnType<typeof createWriteStream>);
vi.mocked(spawn).mockImplementation(() => {
setImmediate(() => {
if (failure === "stdout") stdout.destroy(error);
if (failure === "sink") sink.destroy(error);
if (failure === "spawn") child.emit("error", error);
if (failure === "exit") {
stderr.write(error.message);
stdout.end();
child.emit("close", 1);
}
});
return child as unknown as ReturnType<typeof spawn>;
});
vi.mocked(execFile).mockImplementation((...args: unknown[]) => {
const callback = args.at(-1) as (error: null, stdout: string, stderr: string) => void;
callback(null, "", "");
return {} as ReturnType<typeof execFile>;
});
await expect(restoreWorkspaceFromSshExecution({
spec: {
host: "fixture.invalid", username: "fixture", port: 22,
remoteCwd: "/remote", remoteWorkspacePath: "/remote",
privateKey: null, knownHosts: null, strictHostKeyChecking: true,
},
localDir: "/local",
baselineSnapshot: { entries: new Map(), exclude: [] },
restoreGitHistory: true,
onProgress: async () => {},
})).rejects.toThrow(error.message);
expect(stdout.destroyed).toBe(true);
expect(sink.destroyed).toBe(true);
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
expect(vi.mocked(execFile).mock.calls.some((call) => (call[1] as string[]).includes("fetch"))).toBe(false);
});
it.each([false, true])("drains SSH export before fetching with progress=%s", async (withProgress) => {
const chunks = Array.from({ length: 16 }, (_, index) => Buffer.alloc(64 * 1024, index));
const received: Buffer[] = [];
let fetched: Buffer | undefined;
const sink = new Writable({
highWaterMark: 1024,
write(chunk, _encoding, callback) {
setTimeout(() => { received.push(Buffer.from(chunk)); callback(); }, 2);
},
});
vi.mocked(createWriteStream).mockReturnValue(sink as ReturnType<typeof createWriteStream>);
vi.mocked(spawn).mockImplementation(() => {
const child = Object.assign(new EventEmitter(), {
stdout: Readable.from(chunks), stderr: new PassThrough(), kill: vi.fn(),
});
child.stdout.on("end", () => child.emit("close", 0));
return child as unknown as ReturnType<typeof spawn>;
});
vi.mocked(execFile).mockImplementation((...args: unknown[]) => {
const argv = args[1] as string[];
const callback = args.at(-1) as (error: Error | null, stdout: string, stderr: string) => void;
if (argv.includes("fetch")) {
fetched = Buffer.concat(received);
callback(new Error("stop after fetch"), "", "");
} else {
callback(null, "", "");
}
return {} as ReturnType<typeof execFile>;
});
await expect(restoreWorkspaceFromSshExecution({
spec: {
host: "fixture.invalid", username: "fixture", port: 22,
remoteCwd: "/remote", remoteWorkspacePath: "/remote",
privateKey: null, knownHosts: null, strictHostKeyChecking: true,
},
localDir: "/local",
baselineSnapshot: { entries: new Map(), exclude: [] },
restoreGitHistory: true,
...(withProgress ? { onProgress: async () => {} } : {}),
})).rejects.toThrow("stop after fetch");
expect(fetched).toEqual(Buffer.concat(chunks));
});
@@ -0,0 +1,109 @@
import { execFile, spawn } from "node:child_process";
import { createWriteStream } from "node:fs";
import { randomBytes } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
spawn: vi.fn((file: string, args: string[], options: Parameters<typeof actual.spawn>[2]) =>
file === "ssh"
? actual.spawn("sh", ["-c", args.at(-1)!], options)
: actual.spawn(file, args, options)),
execFile: vi.fn((file: string, args: string[], options: object, callback: Parameters<typeof actual.execFile>[3]) =>
file === "ssh"
? actual.execFile("sh", ["-c", args.at(-1)!], options, callback)
: actual.execFile(file, args, options, callback)),
};
});
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
return {
...actual,
createWriteStream: vi.fn((...args: Parameters<typeof actual.createWriteStream>) => {
const sink = actual.createWriteStream(...args);
const write = sink._write.bind(sink);
const writev = sink._writev!.bind(sink);
sink._write = (...writeArgs) => { setTimeout(() => write(...writeArgs), 2); };
sink._writev = (...writeArgs) => { setTimeout(() => writev(...writeArgs), 2); };
return sink;
}),
};
});
import { restoreWorkspaceFromSshExecution } from "./ssh.js";
const run = promisify(execFile);
const roots: string[] = [];
afterEach(async () => {
for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true });
});
describe("SSH git export", () => {
it.each(["exit", "source", "sink"] as const)("rejects an export on %s failure", async (failure) => {
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
const error = new Error("transfer failed");
vi.mocked(spawn).mockImplementationOnce((_file, _args, options) => {
const child = actual.spawn("sh", ["-c", failure === "exit" ? "printf 'export failed' >&2; exit 7" : "printf payload"], options);
if (failure === "source") queueMicrotask(() => child.stdout!.destroy(error));
return child;
});
if (failure === "sink") {
const fs = await vi.importActual<typeof import("node:fs")>("node:fs");
vi.mocked(createWriteStream).mockImplementationOnce((...args) => {
const sink = fs.createWriteStream(...args);
queueMicrotask(() => sink.destroy(error));
return sink;
});
}
await expect(restoreWorkspaceFromSshExecution({
spec: {
host: "fixture.invalid", username: "fixture", port: 22,
remoteCwd: "/remote", remoteWorkspacePath: "/remote",
privateKey: null, knownHosts: null, strictHostKeyChecking: false,
},
localDir: "/unused",
baselineSnapshot: { exclude: [], entries: new Map() },
restoreGitHistory: true,
onProgress: async () => {},
})).rejects.toEqual(failure === "exit" ? new Error("export failed") : error);
});
it.each([false, true])("drains the local bundle under backpressure with progress=%s", async (withProgress) => {
const root = await mkdtemp(path.join(os.tmpdir(), "ssh-export-"));
roots.push(root);
const remoteDir = path.join(root, "remote");
const localDir = path.join(root, "local");
await mkdir(remoteDir);
await run("git", ["init", remoteDir]);
const contents = randomBytes(2 * 1024 * 1024);
await writeFile(path.join(remoteDir, "payload"), contents);
await run("git", ["-C", remoteDir, "add", "payload"]);
await run("git", ["-C", remoteDir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "payload"]);
await run("git", ["clone", remoteDir, localDir]);
await writeFile(path.join(remoteDir, "updated"), "remote commit\n");
await run("git", ["-C", remoteDir, "add", "updated"]);
await run("git", ["-C", remoteDir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "update"]);
await restoreWorkspaceFromSshExecution({
spec: {
host: "fixture.invalid", username: "fixture", port: 22,
remoteCwd: remoteDir, remoteWorkspacePath: remoteDir,
privateKey: null, knownHosts: null, strictHostKeyChecking: false,
},
localDir, remoteDir,
...(withProgress ? { onProgress: async () => {} } : {}),
});
expect((await run("git", ["-C", localDir, "rev-parse", "HEAD"])).stdout)
.toBe((await run("git", ["-C", remoteDir, "rev-parse", "HEAD"])).stdout);
expect(await readFile(path.join(localDir, "payload"))).toEqual(contents);
expect(await readFile(path.join(localDir, "updated"), "utf8")).toBe("remote commit\n");
}, 30_000);
});
+27 -14
View File
@@ -4,7 +4,7 @@ import { constants as fsConstants, createReadStream, createWriteStream, promises
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { Transform } from "node:stream";
import { pipeline, Transform } from "node:stream";
import type { CommandManagedRuntimeRunner } from "./command-managed-runtime.js";
import {
createUnrelatedHistoryGraftCommit,
@@ -731,36 +731,49 @@ async function streamSshToLocalFile(input: {
let sshStderr = "";
let settled = false;
let sshClosed = false;
let streamFinished = false;
const settle = () => {
if (settled || !sshClosed || !streamFinished) return;
settled = true;
resolve();
};
const fail = (error: Error) => {
if (settled) return;
settled = true;
ssh.kill("SIGTERM");
ssh.stdout.destroy();
input.progress?.counter.destroy();
sink.destroy();
reject(error);
};
const onStreamFinished = (error: NodeJS.ErrnoException | null) => {
if (error) {
fail(error);
return;
}
streamFinished = true;
settle();
};
if (input.progress) {
input.progress.counter.on("error", fail);
ssh.stdout?.pipe(input.progress.counter).pipe(sink);
pipeline(ssh.stdout, input.progress.counter, sink, onStreamFinished);
} else {
ssh.stdout?.pipe(sink);
pipeline(ssh.stdout, sink, onStreamFinished);
}
ssh.stderr?.on("data", (chunk) => {
sshStderr += String(chunk);
});
ssh.on("error", fail);
sink.on("error", fail);
ssh.on("close", (code) => {
sink.end(() => {
if (settled) return;
settled = true;
if ((code ?? 0) !== 0) {
reject(new Error(sshStderr.trim() || `ssh exited with code ${code ?? -1}`));
return;
}
resolve();
});
if ((code ?? 0) !== 0) {
fail(new Error(sshStderr.trim() || `ssh exited with code ${code ?? -1}`));
return;
}
sshClosed = true;
settle();
});
}).finally(auth.cleanup);
}