mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 09:41:04 +01:00
DottaandPaperclip
225dd396c5
Verify ownership before launcher cleanup when browser teardown is interrupted. Co-Authored-By: Paperclip <noreply@paperclip.ing>
200 lines
14 KiB
TypeScript
200 lines
14 KiB
TypeScript
/** Explicit paid/provider stress campaign. Never runs under pnpm test. */
|
|
import assert from "node:assert/strict";
|
|
import { randomUUID } from "node:crypto";
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import plugin from "../../packages/plugins/sandbox-providers/exe-dev/src/plugin.js";
|
|
import { openSsh, ssh, quote } from "../../packages/plugins/sandbox-providers/exe-dev/src/transport.js";
|
|
import { prepareAdapterExecutionTargetRuntime, type AdapterSandboxExecutionTarget } from "../../packages/adapter-utils/src/execution-target.js";
|
|
import type { PluginEnvironmentLease } from "../../packages/plugins/sdk/src/protocol.js";
|
|
|
|
if (process.env.EXE_DEV_LIVE_SMOKE !== "1") throw new Error("Set EXE_DEV_LIVE_SMOKE=1 to authorize this real VM campaign");
|
|
const image = process.env.PAPERCLIP_E2E_EXE_IMAGE;
|
|
const key = process.env.EXE_DEV_SSH_PRIVATE_KEY;
|
|
if (!image || !key) throw new Error("PAPERCLIP_E2E_EXE_IMAGE and EXE_DEV_SSH_PRIVATE_KEY are required");
|
|
const campaign = randomUUID().slice(0, 12);
|
|
const vmName = `paperclip-stress-${campaign}`;
|
|
const config = { mode: "create", vmName, image, sshPrivateKey: key, reuseLease: true, timeoutMs: 300000 };
|
|
const transport = { ...config, strictHostKeyChecking: "accept-new" as const };
|
|
const scope = { companyId: randomUUID(), environmentId: randomUUID(), driverKey: "exe-dev", config };
|
|
const hooks = plugin.definition;
|
|
const leases: PluginEnvironmentLease[] = [];
|
|
const checks: { name: string; durationMs: number }[] = [];
|
|
const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-exe-stress-"));
|
|
const outputPath = process.env.EXE_DEV_EVIDENCE_PATH ?? path.join(temporary, "result.json");
|
|
async function check(name: string, run: () => Promise<void>) {
|
|
const started = Date.now(); await run(); checks.push({ name, durationMs: Date.now() - started }); console.log(`PASS ${name}`);
|
|
}
|
|
async function execute(lease: PluginEnvironmentLease, script: string, timeoutMs = 30000) {
|
|
return hooks.onEnvironmentExecute!({ ...scope, lease, command: "sh", args: ["-c", script], timeoutMs });
|
|
}
|
|
let failure: string | undefined;
|
|
try {
|
|
await check("compatible image and initial durable VM", async () => {
|
|
leases.push(await hooks.onEnvironmentAcquireLease!({ ...scope, runId: randomUUID(), agentId: "agent-0" }));
|
|
assert.equal(leases[0].metadata?.vmName, vmName);
|
|
});
|
|
await check("every bundled legacy CLI starts without credentials", async () => {
|
|
const result = await execute(leases[0], "set -e; for cli in codex claude opencode grok gemini kimi hermes cursor-agent gh; do timeout 45 \"$cli\" --version; done", 300000);
|
|
assert.equal(result.exitCode, 0, result.stderr);
|
|
});
|
|
const resourceBinding = leases[0].metadata?.environmentResourceBinding as NonNullable<Parameters<NonNullable<typeof hooks.onEnvironmentAcquireLease>>[0]["resourceBinding"]>;
|
|
await check("eight independent leases share the bound VM", async () => {
|
|
leases.push(...await Promise.all(Array.from({ length: 7 }, (_, i) => hooks.onEnvironmentAcquireLease!({ ...scope, resourceBinding, runId: randomUUID(), agentId: `agent-${i + 1}` }))));
|
|
assert.equal(new Set(leases.map((l) => l.providerLeaseId)).size, 8);
|
|
assert.equal(new Set(leases.map((l) => l.metadata?.remoteHome)).size, 8);
|
|
assert.equal(new Set(leases.map((l) => l.metadata?.bindingId)).size, 1);
|
|
});
|
|
await check("64 concurrent commands preserve stdin, output, homes, and exit codes", async () => {
|
|
const results = await Promise.all(Array.from({ length: 64 }, async (_, i) => {
|
|
const lease = leases[i % leases.length];
|
|
const result = await hooks.onEnvironmentExecute!({ ...scope, lease, command: "node", args: ["-e", `let x='';process.stdin.setEncoding('utf8');process.stdin.on('data',d=>x+=d);process.stdin.on('end',()=>{require('fs').writeFileSync(process.env.HOME+'/result-${i}',x);process.stdout.write(x);process.exit(${i % 3})})`], stdin: `task-${i}-é-🚀\n`.repeat(500), timeoutMs: 60000 });
|
|
assert.equal(result.stdout, `task-${i}-é-🚀\n`.repeat(500)); assert.equal(result.exitCode, i % 3);
|
|
return result;
|
|
}));
|
|
assert.equal(results.length, 64);
|
|
});
|
|
await check("large output is completely drained before the exit receipt", async () => {
|
|
const result = await execute(leases[0], "node -e \"let pending=2;const done=()=>{if(--pending===0)process.exit(0)};process.stdout.write('x'.repeat(8*1024*1024),done);process.stderr.write('tail'.repeat(256*1024),done)\"");
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(result.stdout.length, 8 * 1024 * 1024);
|
|
assert.equal(result.stderr, "tail".repeat(256 * 1024));
|
|
const started = Date.now();
|
|
const inherited = await execute(leases[0], "node -e \"require('child_process').spawn('sleep',['10'],{stdio:['ignore',process.stdout,process.stderr]}).unref()\"");
|
|
assert.equal(inherited.exitCode, 0);
|
|
assert.ok(Date.now() - started < 8000, "a detached child's inherited pipes must not hold the command open");
|
|
});
|
|
await check("SSH loss does not kill remote work or imply a termination receipt", async () => {
|
|
const root = String(leases[0].metadata?.remoteCwd).replace(/\/workspace$/, "");
|
|
const { child } = await openSsh(transport, `${vmName}.exe.xyz`, `node /opt/paperclip-exe/lease.mjs call ${quote(root)}`);
|
|
let ready = false;
|
|
child.stdout.on("data", () => { ready = true; }); child.stderr.resume();
|
|
child.stdin.write(JSON.stringify({ type: "start", command: "sh", args: ["-c", "printf started; sleep 2; printf completed > survived-disconnect"], timeoutMs: 10000 }) + "\n");
|
|
const deadline = Date.now() + 15000;
|
|
while (!ready && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 100));
|
|
assert.ok(ready, "remote command started");
|
|
child.kill("SIGKILL");
|
|
const result = await execute(leases[0], "sleep 3; cat survived-disconnect");
|
|
assert.equal(result.stdout, "completed");
|
|
});
|
|
await check("workspace seed, remote edits between runs, and copyback", async () => {
|
|
const local = path.join(temporary, "workspace"); await mkdir(local);
|
|
await writeFile(path.join(local, "initial.txt"), "host seed\n");
|
|
const lease = leases[1];
|
|
const target: AdapterSandboxExecutionTarget = { kind: "remote" as const, transport: "sandbox" as const, providerKey: "exe-dev", remoteCwd: String(lease.metadata?.remoteCwd), runner: {
|
|
execute: async (command) => { const result = await hooks.onEnvironmentExecute!({ ...scope, lease, ...command }); return { ...result, signal: result.signal ?? null, pid: null, startedAt: null }; },
|
|
} };
|
|
const first = await prepareAdapterExecutionTargetRuntime({ runId: randomUUID(), target, adapterKey: "stress", workspaceLocalDir: local });
|
|
await first.restoreWorkspace();
|
|
await execute(lease, "printf 'changed remotely\\n' > initial.txt; printf 'durable untracked\\n' > remote-only.txt");
|
|
const next = await prepareAdapterExecutionTargetRuntime({ runId: randomUUID(), target, adapterKey: "stress", workspaceLocalDir: local });
|
|
assert.equal((await execute(lease, "cat initial.txt")).stdout, "changed remotely\n");
|
|
await next.restoreWorkspace();
|
|
assert.equal(await readFile(path.join(local, "initial.txt"), "utf8"), "changed remotely\n");
|
|
assert.equal(await readFile(path.join(local, "remote-only.txt"), "utf8"), "durable untracked\n");
|
|
});
|
|
await check("normal release and worker restart preserve disk", async () => {
|
|
const lease = leases[1];
|
|
await hooks.onEnvironmentReleaseLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata });
|
|
await hooks.onShutdown!();
|
|
const resumed = await hooks.onEnvironmentResumeLease!({ ...scope, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata });
|
|
assert.equal(resumed.providerLeaseId, lease.providerLeaseId);
|
|
assert.equal((await execute(resumed, "cat remote-only.txt")).stdout, "durable untracked\n");
|
|
});
|
|
await check("verified copyback seeds a fresh lease and preserves unrelated host edits", async () => {
|
|
const local = path.join(temporary, "workspace");
|
|
const lease = leases[7];
|
|
const target: AdapterSandboxExecutionTarget = { kind: "remote", transport: "sandbox", providerKey: "exe-dev", remoteCwd: String(lease.metadata?.remoteCwd), runner: {
|
|
execute: async (command) => { const result = await hooks.onEnvironmentExecute!({ ...scope, lease, ...command }); return { ...result, signal: result.signal ?? null, pid: null, startedAt: null }; },
|
|
} };
|
|
const restored = await prepareAdapterExecutionTargetRuntime({ runId: randomUUID(), target, adapterKey: "stress", workspaceLocalDir: local });
|
|
assert.equal((await execute(lease, "cat remote-only.txt")).stdout, "durable untracked\n");
|
|
await writeFile(path.join(local, "host-only.txt"), "host collaboration\n");
|
|
await writeFile(path.join(local, "initial.txt"), "competing host change\n");
|
|
await execute(lease, "printf 'remote final\\n' > initial.txt");
|
|
await restored.restoreWorkspace();
|
|
assert.equal(await readFile(path.join(local, "host-only.txt"), "utf8"), "host collaboration\n");
|
|
// Existing file copyback gives changed remote files precedence at the same
|
|
// path. This is deliberately recorded rather than advertised as a merge.
|
|
assert.equal(await readFile(path.join(local, "initial.txt"), "utf8"), "remote final\n");
|
|
assert.equal((await execute(leases[1], "cat initial.txt")).stdout, "changed remotely\n");
|
|
});
|
|
await check("timeout returns evidence and independent control commands still work", async () => {
|
|
const result = await execute(leases[2], "sleep 30", 200);
|
|
assert.equal(result.timedOut, true);
|
|
assert.equal((await execute(leases[2], "printf healthy")).stdout, "healthy");
|
|
});
|
|
await check("cancel one lease while another continues", async () => {
|
|
const pending = execute(leases[3], "sleep 60").catch((error: Error) => error);
|
|
const other = execute(leases[4], "sleep 3; printf survived");
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
const receipt = await hooks.onEnvironmentReleaseLease!({ ...scope, providerLeaseId: leases[3].providerLeaseId, leaseMetadata: leases[3].metadata, cancelActiveWork: true });
|
|
assert.equal(receipt?.state, "stopped");
|
|
await pending; assert.equal((await other).stdout, "survived");
|
|
});
|
|
await check("lease teardown kills a descendant that escapes its process group", async () => {
|
|
const lease = leases[6];
|
|
await execute(lease, "node -e \"const c=require('child_process').spawn('sleep',['120'],{detached:true,stdio:'ignore'});require('fs').writeFileSync('escaped.pid',String(c.pid));c.unref()\"");
|
|
const pid = Number((await execute(lease, "cat escaped.pid")).stdout.trim());
|
|
assert.ok(Number.isInteger(pid) && pid > 1);
|
|
await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata });
|
|
assert.equal((await execute(leases[4], `if kill -0 ${pid} 2>/dev/null; then printf alive; else printf stopped; fi`)).stdout, "stopped");
|
|
});
|
|
await check("separately managed HTTP service survives lease teardown", async () => {
|
|
const unit = `paperclip-preview-${campaign}`;
|
|
const host = `${vmName}.exe.xyz`;
|
|
await ssh(transport, host, `sudo -n systemd-run --unit=${quote(unit)} --uid=exedev --collect node -e ${quote("require('http').createServer((q,s)=>s.end('preview-alive')).listen(8000,'0.0.0.0')")}`);
|
|
await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: leases[5].providerLeaseId, leaseMetadata: leases[5].metadata });
|
|
assert.equal((await ssh(transport, host, "curl -fsS http://127.0.0.1:8000")).trim(), "preview-alive");
|
|
const response = await fetch(`https://${host}`, { redirect: "manual" });
|
|
assert.ok([302, 303, 307, 308, 401, 403].includes(response.status), `Preview must remain private; got ${response.status}`);
|
|
if (response.status < 400) {
|
|
const login = new URL(response.headers.get("location")!, `https://${host}`);
|
|
assert.equal(login.origin, `https://${host}`);
|
|
assert.equal(login.pathname, "/__exe.dev/login");
|
|
}
|
|
await ssh(transport, host, `sudo -n systemctl stop ${quote(unit)}`);
|
|
});
|
|
const soakMs = Number(process.env.EXE_DEV_SOAK_MS ?? 0);
|
|
if (soakMs > 0) await check(`idle persistence after ${soakMs}ms`, async () => {
|
|
await new Promise((resolve) => setTimeout(resolve, soakMs));
|
|
assert.equal((await execute(leases[1], "cat remote-only.txt")).stdout, "durable untracked\n");
|
|
});
|
|
await check("VM reboot preserves disk and resumed lease identity", async () => {
|
|
const host = `${vmName}.exe.xyz`;
|
|
const previousBoot = await ssh(transport, host, "cat /proc/sys/kernel/random/boot_id");
|
|
await ssh(transport, host, "sudo -n systemctl reboot").catch(() => {});
|
|
const deadline = Date.now() + 90000;
|
|
let rebooted = false;
|
|
while (Date.now() < deadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
try { if (await ssh({ ...transport, timeoutMs: 10000 }, host, "cat /proc/sys/kernel/random/boot_id") !== previousBoot) { rebooted = true; break; } } catch {}
|
|
}
|
|
assert.ok(rebooted, "VM completed reboot");
|
|
const lease = leases[1];
|
|
const resumed = await hooks.onEnvironmentResumeLease!({ ...scope, providerLeaseId: lease.providerLeaseId!, leaseMetadata: lease.metadata });
|
|
assert.equal(resumed.metadata?.bindingId, lease.metadata?.bindingId);
|
|
assert.equal((await execute(resumed, "cat remote-only.txt")).stdout, "durable untracked\n");
|
|
});
|
|
} catch (error) {
|
|
failure = error instanceof Error ? error.stack : String(error);
|
|
} finally {
|
|
for (const lease of leases) {
|
|
try { await hooks.onEnvironmentDestroyLease!({ ...scope, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata }); }
|
|
catch (error) { failure ??= `Lease cleanup failed: ${String(error)}`; }
|
|
}
|
|
// Only this explicitly generated campaign VM is removable. Durable production
|
|
// lifecycle hooks never call rm. Missing VMs are already cleaned up.
|
|
try {
|
|
const list = JSON.parse(await ssh(transport, "exe.dev", "ls --json"));
|
|
if (list.vms.some((vm: Record<string, unknown>) => vm.vm_name === vmName)) await ssh(transport, "exe.dev", `rm --json ${quote(vmName)}`);
|
|
const after = JSON.parse(await ssh(transport, "exe.dev", "ls --json"));
|
|
assert.ok(!after.vms.some((vm: Record<string, unknown>) => vm.vm_name === vmName));
|
|
} catch (error) { failure ??= `VM cleanup failed: ${String(error)}`; }
|
|
await hooks.onShutdown?.();
|
|
await writeFile(outputPath, JSON.stringify({ campaign, image, vmName, checks, failure, finishedAt: new Date().toISOString() }, null, 2), { mode: 0o600 });
|
|
if (failure) { console.error(failure); process.exitCode = 1; }
|
|
console.log(`Evidence: ${outputPath}`);
|
|
}
|