mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 20:21:50 +01:00
Add sandbox work-folder persistence and shared lifecycle
Introduce company-scoped task, agent, user and project files, recoverable trash, object-backed repository checkpoints, and a shared 180-second lifecycle for sandbox execution. Wire file browsing and save status into the owner surfaces. This is an implementation checkpoint; staging acceptance and the complete merge gate remain outstanding.
This commit is contained in:
1 parent
392ab26b1e
commit
a13bb6b395
51 files changed
+176079
-37
No files matched your search
@@ -444,7 +444,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
@@ -4637,7 +4639,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
@@ -4982,7 +4986,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
@@ -5748,7 +5754,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
@@ -5903,7 +5911,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
@@ -6064,7 +6074,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
version: 2,
|
||||
responsibleUserId: null,
|
||||
issueId: null,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
const exec = promisify(execFile);
|
||||
export const localTestWorkFolderRunner: CommandManagedRuntimeRunner = {
|
||||
async execute(input) {
|
||||
try {
|
||||
const { stdout, stderr } = await exec(input.command, input.args ?? [], { cwd: input.cwd,
|
||||
env: { ...process.env, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 });
|
||||
return { stdout, stderr, exitCode: 0, signal: null, timedOut: false };
|
||||
} catch (error) {
|
||||
const value = error as Error & { stdout?: string; stderr?: string };
|
||||
return { stdout: value.stdout ?? "", stderr: value.stderr ?? value.message, exitCode: 1, signal: null, timedOut: false };
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,126 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { agents, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, projects, projectWorkspaces, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { prepareSandboxWorkFolders } from "../services/sandbox-work-folders.js";
|
||||
import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "../services/work-folder-retention.js";
|
||||
import { workFolderService } from "../services/work-folders.js";
|
||||
import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js";
|
||||
const exec = promisify(execFile);
|
||||
|
||||
describe("shared sandbox work-folder lifecycle", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let root: string;
|
||||
let storage: ReturnType<typeof createLocalDiskStorageProvider>;
|
||||
const companyId = randomUUID(), agentId = randomUUID(), projectId = randomUUID(), taskId = randomUUID(), environmentId = randomUUID();
|
||||
const active: Array<Awaited<ReturnType<typeof prepareSandboxWorkFolders>>> = [];
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-sandbox-folders-");
|
||||
db = createDb(database.connectionString);
|
||||
root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-folders-")));
|
||||
storage = createLocalDiskStorageProvider(path.join(root, "bucket"));
|
||||
await db.insert(companies).values({ id: companyId, name: "Sandbox folder tests" });
|
||||
await db.insert(environments).values({ id: environmentId, name: "Test sandbox", driver: "sandbox", config: {} });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Agent" });
|
||||
await db.insert(projects).values({ id: projectId, companyId, name: "Project" });
|
||||
await db.insert(issues).values({ id: taskId, companyId, projectId, title: "Task", assigneeAgentId: agentId });
|
||||
for (const name of ["repo-one", "repo-two"]) {
|
||||
const source = path.join(root, name);
|
||||
await exec("git", ["init", source]);
|
||||
await fs.writeFile(path.join(source, "tracked"), "initial\n");
|
||||
await fs.symlink("tracked", path.join(source, "link"));
|
||||
await exec("git", ["-C", source, "add", "."]);
|
||||
await exec("git", ["-C", source, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]);
|
||||
await db.insert(projectWorkspaces).values({ companyId, projectId, name, repoUrl: source, sourceType: "git_repo", isPrimary: name === "repo-one" });
|
||||
}
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
for (const run of active) await run.stop().catch(() => {});
|
||||
await database?.cleanup(); if (root) await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
async function prepare(home: string, leaseId: string, physicalId = leaseId) {
|
||||
await fs.mkdir(home, { recursive: true });
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" });
|
||||
const lease = { id: leaseId, companyId, environmentId, provider: "test", providerLeaseId: physicalId };
|
||||
await db.insert(environmentLeases).values({ ...lease, heartbeatRunId: runId }).onConflictDoUpdate({ target: environmentLeases.id, set: { heartbeatRunId: runId } });
|
||||
const run = await prepareSandboxWorkFolders({ db, companyId, agentId, projectId, taskId, runId,
|
||||
responsibleUserId: null, storage, sandboxKey: workFolderSandboxKey(lease), target: { kind: "remote", transport: "sandbox", leaseId, remoteCwd: home,
|
||||
runner: { execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } });
|
||||
active.push(run); return run;
|
||||
}
|
||||
it("reuses clones and restores saved unpushed work, staged changes, and task files after losing the sandbox", async () => {
|
||||
const home = path.join(root, "sandbox");
|
||||
const leaseId = randomUUID();
|
||||
const first = await prepare(home, leaseId);
|
||||
expect(first.home).toBe(home);
|
||||
expect(first.manifest.repositories).toHaveLength(2);
|
||||
expect(first.primaryRepo).toBe(path.join(home, "repos/repo-one"));
|
||||
await fs.writeFile(path.join(home, "task/report.md"), "durable task file");
|
||||
const repo = first.primaryRepo;
|
||||
await fs.writeFile(path.join(repo, "tracked"), "committed\n");
|
||||
await exec("git", ["-C", repo, "add", "."]);
|
||||
await exec("git", ["-C", repo, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "unpushed"]);
|
||||
const expectedHead = (await exec("git", ["-C", repo, "rev-parse", "HEAD"])).stdout.trim();
|
||||
await fs.writeFile(path.join(repo, "tracked"), "staged\n");
|
||||
await exec("git", ["-C", repo, "add", "tracked"]);
|
||||
await fs.writeFile(path.join(repo, "tracked"), "unstaged\n");
|
||||
await fs.writeFile(path.join(repo, "untracked"), "untracked\n");
|
||||
await first.stop(); active.splice(active.indexOf(first), 1);
|
||||
const warm = await prepare(home, randomUUID(), leaseId);
|
||||
expect(await fs.readFile(path.join(repo, "tracked"), "utf8")).toBe("unstaged\n");
|
||||
await warm.stop(); active.splice(active.indexOf(warm), 1);
|
||||
await fs.rm(home, { recursive: true });
|
||||
const restored = await prepare(path.join(root, "replacement"), randomUUID());
|
||||
expect(await fs.readFile(path.join(restored.home, "task/report.md"), "utf8")).toBe("durable task file");
|
||||
expect((await exec("git", ["-C", restored.primaryRepo, "rev-parse", "HEAD"])).stdout.trim()).toBe(expectedHead);
|
||||
expect((await exec("git", ["-C", restored.primaryRepo, "show", ":tracked"])).stdout).toBe("staged\n");
|
||||
expect(await fs.readFile(path.join(restored.primaryRepo, "tracked"), "utf8")).toBe("unstaged\n");
|
||||
expect(await fs.readFile(path.join(restored.primaryRepo, "untracked"), "utf8")).toBe("untracked\n");
|
||||
expect(await fs.readlink(path.join(restored.primaryRepo, "link"))).toBe("tracked");
|
||||
await restored.stop(); active.splice(active.indexOf(restored), 1);
|
||||
}, 120_000);
|
||||
it("does not let an unchanged stale shared file overwrite a newer durable value", async () => {
|
||||
const svc = workFolderService(db, storage);
|
||||
const folder = await svc.ensure({ companyId, scope: "project", ownerId: projectId });
|
||||
await svc.write(folder, { path: "shared.md", body: Buffer.from("first"), operationId: randomUUID() });
|
||||
const run = await prepare(path.join(root, "stale-sandbox"), randomUUID());
|
||||
await svc.write(folder, { path: "shared.md", body: Buffer.from("newer"), operationId: randomUUID() });
|
||||
await run.stop(); active.splice(active.indexOf(run), 1);
|
||||
const result = await svc.content(folder, "shared.md");
|
||||
const chunks = []; for await (const chunk of result.stream) chunks.push(Buffer.from(chunk));
|
||||
expect(Buffer.concat(chunks).toString()).toBe("newer");
|
||||
}, 120_000);
|
||||
it("retains the only working copy until a final checkpoint succeeds", async () => {
|
||||
const leaseId = randomUUID();
|
||||
const run = await prepare(path.join(root, "retained-sandbox"), leaseId);
|
||||
await fs.writeFile(path.join(run.home, "task/pending"), "recover me");
|
||||
await run.flush();
|
||||
expect(await retainUnsavedWorkFolderLease(db, { id: leaseId, companyId })).toBe(true);
|
||||
await run.stop(); active.splice(active.indexOf(run), 1);
|
||||
expect(await retainUnsavedWorkFolderLease(db, { id: leaseId, companyId })).toBe(false);
|
||||
}, 120_000);
|
||||
it("reconciles file-directory replacements and preserves deleted children in trash", async () => {
|
||||
const svc = workFolderService(db, storage);
|
||||
const folder = await svc.ensure({ companyId, scope: "project", ownerId: projectId });
|
||||
await svc.write(folder, { path: "replace/child", body: Buffer.from("child"), operationId: randomUUID() });
|
||||
const leaseId = randomUUID();
|
||||
const home = path.join(root, "replacement-kinds");
|
||||
const run = await prepare(home, leaseId);
|
||||
await fs.rm(path.join(home, "project/replace"), { recursive: true });
|
||||
await fs.writeFile(path.join(home, "project/replace"), "now a file");
|
||||
await run.stop(); active.splice(active.indexOf(run), 1);
|
||||
expect((await svc.list(folder, { trash: true })).files.map((file) => file.path)).toContain("replace/child");
|
||||
await svc.write(folder, { path: "replace", kind: "directory", replaceKind: true, operationId: randomUUID() });
|
||||
const resumed = await prepare(home, randomUUID(), leaseId);
|
||||
expect((await fs.stat(path.join(home, "project/replace"))).isDirectory()).toBe(true);
|
||||
await resumed.stop(); active.splice(active.indexOf(resumed), 1);
|
||||
}, 120_000);
|
||||
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { startWorkFolderCheckpointer } from "../services/work-folder-checkpointer.js";
|
||||
|
||||
describe("shared work folder checkpoint cadence", () => {
|
||||
afterEach(() => vi.useRealTimers());
|
||||
it("waits 180 seconds, does not overlap, and flushes after the last in-flight save", async () => {
|
||||
vi.useFakeTimers();
|
||||
let finish!: () => void;
|
||||
const checkpoint = vi.fn().mockImplementationOnce(() => new Promise<void>((resolve) => { finish = resolve; }))
|
||||
.mockResolvedValue(undefined);
|
||||
const sync = startWorkFolderCheckpointer({ checkpoint, onError: vi.fn() });
|
||||
await vi.advanceTimersByTimeAsync(179_999);
|
||||
expect(checkpoint).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(checkpoint).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(360_000);
|
||||
expect(checkpoint).toHaveBeenCalledTimes(1);
|
||||
const stopped = sync.stop();
|
||||
finish();
|
||||
await stopped;
|
||||
expect(checkpoint).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(180_000);
|
||||
expect(checkpoint).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("reports periodic errors and fails completion if the final save fails", async () => {
|
||||
vi.useFakeTimers();
|
||||
const error = new Error("Storage unavailable");
|
||||
const onError = vi.fn().mockResolvedValue(undefined);
|
||||
const sync = startWorkFolderCheckpointer({ checkpoint: vi.fn().mockRejectedValue(error), onError });
|
||||
await vi.advanceTimersByTimeAsync(180_000);
|
||||
expect(onError).toHaveBeenCalledWith(error);
|
||||
await expect(sync.stop()).rejects.toThrow("Storage unavailable");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, startEmbeddedPostgresTestDatabase, workFolderRuns, type Db } from "@paperclipai/db";
|
||||
import { workFolderRoutes } from "../routes/work-folders.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { workFolderService } from "../services/work-folders.js";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import type { AuthorizationActor } from "../services/authorization.js";
|
||||
|
||||
describe("work folder HTTP ownership and streaming", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db, root: string;
|
||||
let storage: ReturnType<typeof createLocalDiskStorageProvider>;
|
||||
const companyId = randomUUID(), otherCompanyId = randomUUID(), ownerId = randomUUID(), otherUserId = randomUUID(), agentId = randomUUID();
|
||||
const base = `/api/companies/${companyId}/work-folders/user/${ownerId}`;
|
||||
const owner: AuthorizationActor = { type: "board", source: "session", userId: ownerId, companyIds: [companyId] };
|
||||
function app(actor: AuthorizationActor) {
|
||||
const server = express();
|
||||
server.use(express.json());
|
||||
server.use((req, _res, next) => { req.actor = actor as typeof req.actor; next(); });
|
||||
server.use("/api", workFolderRoutes(db, storage));
|
||||
server.use(errorHandler);
|
||||
return server;
|
||||
}
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-work-folder-routes-"); db = createDb(database.connectionString);
|
||||
root = await mkdtemp(path.join(os.tmpdir(), "paperclip-work-folder-http-")); storage = createLocalDiskStorageProvider(root);
|
||||
await db.insert(companies).values([{ id: companyId, name: "Files", issuePrefix: "FIL" }, { id: otherCompanyId, name: "Other", issuePrefix: "OTH" }]);
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Runner" });
|
||||
for (const id of [ownerId, otherUserId]) {
|
||||
await db.insert(authUsers).values({ id, name: id, email: `${id}@example.test`, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: id, membershipRole: "owner", status: "active" });
|
||||
}
|
||||
}, 60_000);
|
||||
afterAll(async () => { await database?.cleanup(); if (root) await rm(root, { recursive: true, force: true }); });
|
||||
it("uploads empty files and streams nested content with safe download headers", async () => {
|
||||
await request(app(owner)).put(`${base}/content?path=empty`).set("Content-Type", "application/octet-stream").send(Buffer.alloc(0)).expect(200);
|
||||
const body = Buffer.from("#!/bin/sh\ntrue\n");
|
||||
await request(app(owner)).put(`${base}/content?path=bin/run`).set("Content-Type", "application/octet-stream")
|
||||
.set("X-File-Executable", "true").send(body).expect(200);
|
||||
const listed = await request(app(owner)).get(base).expect(200);
|
||||
expect(listed.body.files.find((file: { path: string }) => file.path === "empty").byteSize).toBe(0);
|
||||
expect(listed.body.files.find((file: { path: string }) => file.path === "bin/run").executable).toBe(true);
|
||||
const downloaded = await request(app(owner)).get(`${base}/content?path=bin/run`).expect(200);
|
||||
expect(downloaded.body).toEqual(body);
|
||||
expect(downloaded.headers["content-security-policy"]).toContain("sandbox");
|
||||
expect(downloaded.headers["cache-control"]).toBe("private, no-store");
|
||||
});
|
||||
it("denies another company member every private-file operation", async () => {
|
||||
const other = app({ type: "board", source: "session", userId: otherUserId, companyIds: [companyId] });
|
||||
await request(other).get(base).expect(404);
|
||||
await request(other).get(`${base}/content?path=empty`).expect(404);
|
||||
await request(other).put(`${base}/content?path=empty`).set("Content-Type", "application/octet-stream").send("changed").expect(404);
|
||||
for (const action of ["delete", "mkdir", "restore", "purge"]) await request(other).post(`${base}/operations`).send({ action, path: "empty", fileId: randomUUID() }).expect(404);
|
||||
await request(other).get(`${base}/sync`).expect(404);
|
||||
await request(other).post(`${base}/refresh`).send({ runId: randomUUID() }).expect(404);
|
||||
const foreign = app({ type: "agent", source: "agent_key", companyId: otherCompanyId, agentId: randomUUID() });
|
||||
await request(foreign).get(base).expect(404);
|
||||
});
|
||||
it("authorizes a bound running agent and immediately honors membership revocation", async () => {
|
||||
const folder = await workFolderService(db, storage).ensure({ companyId, scope: "user", ownerId });
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: ownerId, status: "running" });
|
||||
await db.insert(workFolderRuns).values({ runId, companyId, manifest: { version: 1, companyId, runId, agentId,
|
||||
taskId: null, responsibleUserId: ownerId, projectId: null, leaseId: randomUUID(), sandboxKey: randomUUID(), home: "/home/daytona",
|
||||
folders: { task: null, agent: null, user: folder.id, project: null }, repositories: [] } });
|
||||
const runApp = app({ type: "agent", source: "agent_jwt", companyId, agentId, runId, onBehalfOfUserId: ownerId });
|
||||
await request(runApp).get(base).expect(200);
|
||||
await db.update(companyMemberships).set({ status: "inactive" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, ownerId)));
|
||||
await request(runApp).get(base).expect(404);
|
||||
await db.update(companyMemberships).set({ status: "active" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, ownerId)));
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
|
||||
await request(runApp).get(base).expect(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import { createHash } from "node:crypto";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { workFolderTransport } from "../services/work-folder-transport.js";
|
||||
import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
describe("sandbox work folder transport with real Node and Git", () => {
|
||||
const roots: string[] = [];
|
||||
const transport = workFolderTransport(localTestWorkFolderRunner);
|
||||
async function root() {
|
||||
const dir = await realpath(await mkdtemp(path.join(os.tmpdir(), "work-folder-io-")));
|
||||
roots.push(dir); return dir;
|
||||
}
|
||||
afterEach(async () => { for (const dir of roots.splice(0)) await rm(dir, { recursive: true, force: true }); });
|
||||
it("streams and atomically publishes files larger than a transfer chunk", async () => {
|
||||
const dir = await root();
|
||||
const staging = await root();
|
||||
const body = Buffer.alloc(700_000, "x");
|
||||
const entry = { path: "nested/file", kind: "file" as const, byteSize: body.length,
|
||||
sha256: createHash("sha256").update(body).digest("hex"), executable: true };
|
||||
await transport.write(dir, staging, entry, Readable.from([body]));
|
||||
expect(await readFile(path.join(dir, entry.path))).toEqual(body);
|
||||
const files = await transport.scan(dir);
|
||||
expect(files.find((file) => file.path === entry.path)).toEqual(entry);
|
||||
});
|
||||
it("rejects links out of a work folder on scan and download", async () => {
|
||||
const dir = await root();
|
||||
const outside = await root();
|
||||
await writeFile(path.join(outside, "credential"), "private");
|
||||
await symlink(path.join(outside, "credential"), path.join(dir, "link"));
|
||||
await expect(transport.scan(dir)).rejects.toThrow("symlink_not_allowed");
|
||||
const stream = transport.read(dir, "link", 7);
|
||||
await expect((async () => { for await (const _chunk of stream) { /* consume */ } })()).rejects.toThrow("symlink_not_allowed");
|
||||
});
|
||||
it("includes uncommitted tracked and nonignored files plus Git state, excluding credentials and caches", async () => {
|
||||
const dir = await root();
|
||||
await exec("git", ["init", dir]);
|
||||
await writeFile(path.join(dir, ".gitignore"), "node_modules/\n");
|
||||
await writeFile(path.join(dir, "tracked"), "initial");
|
||||
await exec("git", ["-C", dir, "add", "."]);
|
||||
await exec("git", ["-C", dir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]);
|
||||
await writeFile(path.join(dir, "tracked"), "unstaged");
|
||||
await writeFile(path.join(dir, "untracked"), "new");
|
||||
await mkdir(path.join(dir, "node_modules"));
|
||||
await writeFile(path.join(dir, "node_modules/cache"), "ignored");
|
||||
const paths = (await transport.scan(dir, true)).map((entry) => entry.path);
|
||||
expect(paths).toContain("tracked");
|
||||
expect(paths).toContain("untracked");
|
||||
expect(paths).toContain(".git/index");
|
||||
expect(paths).toContain(".git/HEAD");
|
||||
expect(paths).not.toContain(".git/config");
|
||||
expect(paths.some((entry) => entry.startsWith("node_modules/"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { Readable } from "node:stream";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, workFolderObjects, workFolders, companies, createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db";
|
||||
import { validateWorkFilePath } from "@paperclipai/shared";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { collectWorkFolderGarbage } from "../services/work-folder-garbage.js";
|
||||
import { workFolderService } from "../services/work-folders.js";
|
||||
import { assertWorkFolderAccess } from "../services/work-folder-access.js";
|
||||
|
||||
describe("durable work folders", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let root: string;
|
||||
let storage: ReturnType<typeof createLocalDiskStorageProvider>;
|
||||
let svc: ReturnType<typeof workFolderService>;
|
||||
const companyId = randomUUID();
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-work-folders-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Work folders test" });
|
||||
root = await mkdtemp(path.join(os.tmpdir(), "paperclip-work-folders-"));
|
||||
storage = createLocalDiskStorageProvider(root);
|
||||
svc = workFolderService(db, storage);
|
||||
}, 60_000);
|
||||
afterAll(async () => { await database?.cleanup(); if (root) await rm(root, { recursive: true, force: true }); });
|
||||
const folder = async () => {
|
||||
const ownerId = randomUUID();
|
||||
await db.insert(agents).values({ id: ownerId, companyId, name: "File owner" });
|
||||
return svc.ensure({ companyId, scope: "agent", ownerId });
|
||||
};
|
||||
async function textContent(f: Awaited<ReturnType<typeof folder>>, filePath: string) {
|
||||
const { stream } = await svc.content(f, filePath);
|
||||
const buffers: Buffer[] = [];
|
||||
for await (const chunk of stream) buffers.push(Buffer.from(chunk));
|
||||
return Buffer.concat(buffers).toString();
|
||||
}
|
||||
it("streams nested executable and empty files into durable storage", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "bin/run", body: Readable.from(["#!/bin/sh\n", "true\n"]), executable: true, operationId: "first" });
|
||||
await svc.write(f, { path: "empty", body: Buffer.alloc(0), operationId: "empty" });
|
||||
expect(await textContent(f, "bin/run")).toBe("#!/bin/sh\ntrue\n");
|
||||
expect(await textContent(f, "empty")).toBe("");
|
||||
expect((await svc.get(f, "bin/run")).executable).toBe(true);
|
||||
expect((await svc.get(f, "bin")).kind).toBe("directory");
|
||||
});
|
||||
it("does not replay an older accepted write over newer content", async () => {
|
||||
const f = await folder();
|
||||
const first = { path: "memory.md", body: Buffer.from("first"), operationId: "first" };
|
||||
await svc.write(f, first);
|
||||
await svc.write(f, { ...first, body: Buffer.from("second"), operationId: "second" });
|
||||
expect(await svc.write(f, first)).toEqual({ applied: false });
|
||||
expect(await textContent(f, "memory.md")).toBe("second");
|
||||
await expect(svc.write(f, { ...first, body: Buffer.from("different") })).rejects.toMatchObject({ status: 409 });
|
||||
});
|
||||
it("retains a deleted copy after the same path is recreated", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "note", body: Buffer.from("deleted"), operationId: "one" });
|
||||
await svc.remove(f, "note", "delete");
|
||||
const [trashed] = (await svc.list(f, { trash: true })).files;
|
||||
await svc.write(f, { path: "note", body: Buffer.from("replacement"), operationId: "two" });
|
||||
await expect(svc.restore(f, trashed!.id, "restore")).rejects.toMatchObject({ status: 409 });
|
||||
await svc.remove(f, "note", "delete-replacement");
|
||||
await svc.restore(f, trashed!.id, "restore");
|
||||
expect(await textContent(f, "note")).toBe("deleted");
|
||||
expect((await svc.list(f, { trash: true })).files).toHaveLength(1);
|
||||
});
|
||||
it("restores a deleted directory as one recoverable subtree", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "notes/nested/one", body: Buffer.from("one"), operationId: "seed" });
|
||||
await svc.remove(f, "notes", "remove-subtree");
|
||||
const trash = (await svc.list(f, { trash: true })).files;
|
||||
await svc.restore(f, trash.find((file) => file.path === "notes")!.id, "restore-subtree");
|
||||
expect(await textContent(f, "notes/nested/one")).toBe("one");
|
||||
expect((await svc.list(f, { trash: true })).files).toHaveLength(0);
|
||||
});
|
||||
it("does not publish interrupted or oversized uploads", async () => {
|
||||
const f = await folder();
|
||||
const body = Readable.from((async function* () { yield Buffer.from("partial"); throw new Error("Disconnected"); })());
|
||||
await expect(svc.write(f, { path: "partial", body, operationId: "partial" })).rejects.toThrow("Disconnected");
|
||||
await expect(svc.write(f, { path: "large", body: Buffer.from("large"), maxBytes: 2, operationId: "large" })).rejects.toMatchObject({ status: 413 });
|
||||
expect((await svc.list(f)).files).toHaveLength(0);
|
||||
});
|
||||
it("serializes conflicting parent/file creation", async () => {
|
||||
const f = await folder();
|
||||
const results = await Promise.allSettled([
|
||||
svc.write(f, { path: "parent", body: Buffer.from("file"), operationId: "parent" }),
|
||||
svc.write(f, { path: "parent/child", body: Buffer.from("child"), operationId: "child" }),
|
||||
]);
|
||||
expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1);
|
||||
expect(results.filter((r) => r.status === "rejected")).toHaveLength(1);
|
||||
});
|
||||
it("collects overwrites and purged trash but keeps recoverable deleted content", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "note", body: Buffer.from("old"), operationId: "old" });
|
||||
const old = await svc.get(f, "note");
|
||||
await svc.write(f, { path: "note", body: Buffer.from("new"), operationId: "new" });
|
||||
const current = await svc.get(f, "note");
|
||||
await svc.remove(f, "note", "trash");
|
||||
await collectWorkFolderGarbage(db, storage);
|
||||
expect((await storage.headObject({ objectKey: old.objectKey! })).exists).toBe(false);
|
||||
expect((await storage.headObject({ objectKey: current.objectKey! })).exists).toBe(true);
|
||||
await svc.purge(f, current.id, "purge");
|
||||
await collectWorkFolderGarbage(db, storage);
|
||||
expect((await storage.headObject({ objectKey: current.objectKey! })).exists).toBe(false);
|
||||
expect((await svc.list(f, { trash: true })).files).toHaveLength(0);
|
||||
});
|
||||
it("removes scoped files after permanent owner deletion", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "note", body: Buffer.from("private"), operationId: "private" });
|
||||
const file = await svc.get(f, "note");
|
||||
await db.delete(agents).where(eq(agents.id, f.ownerId));
|
||||
await collectWorkFolderGarbage(db, storage);
|
||||
expect(await db.select().from(workFolders).where(eq(workFolders.id, f.id))).toHaveLength(0);
|
||||
expect((await storage.headObject({ objectKey: file.objectKey! })).exists).toBe(false);
|
||||
expect(await db.select().from(workFolderObjects).where(and(eq(workFolderObjects.folderId, f.id), eq(workFolderObjects.companyId, companyId)))).toHaveLength(0);
|
||||
});
|
||||
it("hard-denies private user files to another user or an unbound agent", async () => {
|
||||
const owner = { companyId, scope: "user" as const, ownerId: "owner" };
|
||||
await expect(assertWorkFolderAccess(db, { type: "board", source: "local_implicit", userId: "another" }, owner, false)).rejects.toMatchObject({ status: 404 });
|
||||
await expect(assertWorkFolderAccess(db, { type: "agent", source: "agent_key", companyId, agentId: randomUUID() }, owner, false)).rejects.toMatchObject({ status: 404 });
|
||||
await expect(assertWorkFolderAccess(db, { type: "board", source: "local_implicit", userId: "owner" }, owner, true)).resolves.toBeUndefined();
|
||||
});
|
||||
it.each(["../secret", "/absolute", "a/../../x", "a//b", "a\\b", "a\u0000b", ".paperclip-runtime/secret"])("rejects unsafe path %j", (filePath) => {
|
||||
expect(() => validateWorkFilePath(filePath)).toThrow();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user