mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-30 05:16:50 +01:00
Fix live sandbox sync progress fallback
This commit is contained in:
1 parent
639e1715d9
commit
035c701066
3 files changed
+48
-3
No files matched your search
@@ -22,9 +22,10 @@ interface SpawnRunnerHandle {
|
||||
// A runner that actually executes the shell scripts (piping stdin through a real
|
||||
// pipe so multi-MB payloads work) and replays stdout through onLog in several
|
||||
// chunks so the streaming readFile byte-counter is exercised.
|
||||
function makeSpawnRunner(): SpawnRunnerHandle {
|
||||
function makeSpawnRunner(options: { supportsSingleStreamStdinProgress?: boolean } = {}): SpawnRunnerHandle {
|
||||
const calls: Array<{ command: string; args?: string[]; cwd?: string; stdin?: string }> = [];
|
||||
const runner: CommandManagedRuntimeRunner = {
|
||||
supportsSingleStreamStdinProgress: options.supportsSingleStreamStdinProgress,
|
||||
execute: async (input) =>
|
||||
await new Promise<RunProcessResult>((resolve) => {
|
||||
calls.push({ command: input.command, args: input.args, cwd: input.cwd, stdin: input.stdin });
|
||||
@@ -238,7 +239,7 @@ describe("command managed runtime", () => {
|
||||
const payload = Buffer.alloc(3 * 1024 * 1024);
|
||||
for (let i = 0; i < payload.length; i++) payload[i] = i % 256;
|
||||
|
||||
const { runner, calls } = makeSpawnRunner();
|
||||
const { runner, calls } = makeSpawnRunner({ supportsSingleStreamStdinProgress: true });
|
||||
const client = createCommandManagedRuntimeClient({ runner, commandCwd: "/", timeoutMs: 30_000 });
|
||||
|
||||
const progress: Array<{ done: number; total: number | null }> = [];
|
||||
@@ -263,6 +264,37 @@ describe("command managed runtime", () => {
|
||||
expect(progress.at(-1)).toEqual({ done: payload.length, total: payload.length });
|
||||
});
|
||||
|
||||
it("falls back to chunked upload progress when the runner cannot report mid-stream stdin progress", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-write-fallback-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const remotePath = path.join(rootDir, "nested", "payload.bin");
|
||||
|
||||
const payload = Buffer.alloc(12 * 1024 * 1024);
|
||||
for (let i = 0; i < payload.length; i++) payload[i] = i % 256;
|
||||
|
||||
const { runner, calls } = makeSpawnRunner({ supportsSingleStreamStdinProgress: false });
|
||||
const client = createCommandManagedRuntimeClient({ runner, commandCwd: "/", timeoutMs: 30_000 });
|
||||
|
||||
const progress: Array<{ done: number; total: number | null }> = [];
|
||||
await client.writeFile(remotePath, toArrayBuffer(payload), {
|
||||
onProgress: (done, total) => {
|
||||
progress.push({ done, total });
|
||||
},
|
||||
});
|
||||
|
||||
const written = await readFile(remotePath);
|
||||
expect(written.equals(payload)).toBe(true);
|
||||
|
||||
// Provider-backed sandbox runners cannot surface mid-flight progress for a
|
||||
// single stdin RPC, so we intentionally use several large append commands.
|
||||
expect(calls.length).toBeGreaterThan(2);
|
||||
expect(progress.length).toBeGreaterThan(2);
|
||||
for (let i = 1; i < progress.length; i++) {
|
||||
expect(progress[i].done).toBeGreaterThanOrEqual(progress[i - 1].done);
|
||||
}
|
||||
expect(progress.at(-1)).toEqual({ done: payload.length, total: payload.length });
|
||||
});
|
||||
|
||||
it("streams a download through onLog and reports monotonic byte progress to the total", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-read-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
||||
@@ -11,6 +11,13 @@ import type { RunProcessResult } from "./server-utils.js";
|
||||
import type { RuntimeProgressSink } from "./runtime-progress.js";
|
||||
|
||||
export interface CommandManagedRuntimeRunner {
|
||||
/**
|
||||
* True only when `execute({ stdin })` can surface useful in-flight progress
|
||||
* for a single stdin-backed command. Provider-backed sandbox runners usually
|
||||
* complete the entire RPC before returning, so they should leave this false
|
||||
* and let the caller choose a chunked upload path when progress is requested.
|
||||
*/
|
||||
supportsSingleStreamStdinProgress?: boolean;
|
||||
execute(input: {
|
||||
command: string;
|
||||
args?: string[];
|
||||
@@ -100,13 +107,18 @@ export function createCommandManagedRuntimeClient(input: {
|
||||
const body = buffer.toString("base64");
|
||||
const remoteDir = path.posix.dirname(remotePath);
|
||||
const remoteTempPath = `${remotePath}.paperclip-upload`;
|
||||
const canUseSingleStreamProgressPath =
|
||||
!options?.onProgress || input.runner.supportsSingleStreamStdinProgress === true;
|
||||
|
||||
// Primary path: a single round-trip. Stream the entire base64 body to one
|
||||
// `base64 -d` process via stdin, decode straight into a temp file, then
|
||||
// atomically rename into place. This replaces the previous loop that did
|
||||
// one `printf >> tmpfile` shell round-trip per 32 KB — thousands of serial
|
||||
// processes for a large workspace — with exactly one process.
|
||||
if (body.length <= REMOTE_WRITE_SINGLE_STREAM_MAX_BASE64_BYTES) {
|
||||
if (
|
||||
body.length <= REMOTE_WRITE_SINGLE_STREAM_MAX_BASE64_BYTES &&
|
||||
canUseSingleStreamProgressPath
|
||||
) {
|
||||
await options?.onProgress?.(0, total);
|
||||
await runShell(
|
||||
`mkdir -p ${shellQuote(remoteDir)} && ` +
|
||||
|
||||
@@ -75,6 +75,7 @@ export async function resolveEnvironmentExecutionTarget(input: {
|
||||
timeoutMs,
|
||||
runner: input.environmentRuntime && input.lease
|
||||
? {
|
||||
supportsSingleStreamStdinProgress: false,
|
||||
execute: async (commandInput) => {
|
||||
const startedAt = new Date().toISOString();
|
||||
const result = await input.environmentRuntime!.execute({
|
||||
|
||||
Reference in new issue
Block a user