Add workspace file resource API

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-06-09 21:35:48 +00:00
1 parent a0f7d3daba
commit e11a5d1b2d
11 files changed
+1247

No files matched your search

+17
View File
@@ -449,6 +449,14 @@ export type {
WorkspaceOperation,
WorkspaceOperationPhase,
WorkspaceOperationStatus,
WorkspaceFileContent,
WorkspaceFileContentEncoding,
WorkspaceFilePreviewKind,
WorkspaceFileRef,
WorkspaceFileResourceKind,
WorkspaceFileSelector,
WorkspaceFileWorkspaceKind,
ResolvedWorkspaceResource,
WorkspaceRuntimeDesiredState,
WorkspaceRealizationRecord,
WorkspaceRealizationRequest,
@@ -1037,6 +1045,14 @@ export {
executionWorkspaceCloseLinkedIssueSchema,
executionWorkspaceCloseReadinessSchema,
executionWorkspaceCloseReadinessStateSchema,
resolvedWorkspaceResourceSchema,
workspaceFileContentSchema,
workspaceFilePreviewKindSchema,
workspaceFileRefSchema,
workspaceFileResourceKindSchema,
workspaceFileResourceQuerySchema,
workspaceFileSelectorSchema,
workspaceFileWorkspaceKindSchema,
issueDocumentFormatSchema,
issueDocumentKeySchema,
upsertIssueDocumentSchema,
@@ -1065,6 +1081,7 @@ export {
type UpdateIssueWorkProduct,
type CompanyArtifactsQuery,
type UpdateExecutionWorkspace,
type WorkspaceFileResourceQuery,
type IssueDocumentFormat,
type UpsertIssueDocument,
type RestoreIssueDocumentRevision,
+10
View File
@@ -216,6 +216,16 @@ export type {
WorkspaceOperationPhase,
WorkspaceOperationStatus,
} from "./workspace-operation.js";
export type {
WorkspaceFileContent,
WorkspaceFileContentEncoding,
WorkspaceFilePreviewKind,
WorkspaceFileRef,
WorkspaceFileResourceKind,
WorkspaceFileSelector,
WorkspaceFileWorkspaceKind,
ResolvedWorkspaceResource,
} from "./workspace-file-resource.js";
export type {
IssueWorkProduct,
IssueWorkProductType,
@@ -0,0 +1,43 @@
export type WorkspaceFileWorkspaceKind = "execution_workspace" | "project_workspace";
export type WorkspaceFileSelector = "auto" | "execution" | "project";
export type WorkspaceFilePreviewKind = "text" | "image" | "pdf" | "unsupported";
export type WorkspaceFileResourceKind = "file" | "remote_resource";
export type WorkspaceFileContentEncoding = "utf8" | "base64";
export interface WorkspaceFileRef {
kind: "workspace_file";
issueId: string;
workspaceKind: WorkspaceFileWorkspaceKind;
workspaceId: string;
relativePath: string;
line?: number | null;
column?: number | null;
displayPath: string;
}
export interface ResolvedWorkspaceResource {
kind: WorkspaceFileResourceKind;
provider: "local_fs" | "git_worktree" | "remote_managed" | string;
title: string;
displayPath: string;
workspaceLabel: string;
workspaceKind: WorkspaceFileWorkspaceKind;
workspaceId: string;
contentType?: string | null;
byteSize?: number | null;
previewKind: WorkspaceFilePreviewKind;
denialReason?: string | null;
capabilities: {
preview: boolean;
download: false;
listChildren: false;
};
}
export interface WorkspaceFileContent {
resource: ResolvedWorkspaceResource;
content: {
encoding: WorkspaceFileContentEncoding;
data: string;
};
}
+12
View File
@@ -356,6 +356,18 @@ export {
type UpdateExecutionWorkspace,
} from "./execution-workspace.js";
export {
resolvedWorkspaceResourceSchema,
workspaceFileContentSchema,
workspaceFilePreviewKindSchema,
workspaceFileRefSchema,
workspaceFileResourceKindSchema,
workspaceFileResourceQuerySchema,
workspaceFileSelectorSchema,
workspaceFileWorkspaceKindSchema,
type WorkspaceFileResourceQuery,
} from "./workspace-file-resource.js";
export {
createGoalSchema,
updateGoalSchema,
@@ -0,0 +1,51 @@
import { z } from "zod";
export const workspaceFileWorkspaceKindSchema = z.enum(["execution_workspace", "project_workspace"]);
export const workspaceFileSelectorSchema = z.enum(["auto", "execution", "project"]).default("auto");
export const workspaceFilePreviewKindSchema = z.enum(["text", "image", "pdf", "unsupported"]);
export const workspaceFileResourceKindSchema = z.enum(["file", "remote_resource"]);
export const workspaceFileRefSchema = z.object({
kind: z.literal("workspace_file"),
issueId: z.string().uuid(),
workspaceKind: workspaceFileWorkspaceKindSchema,
workspaceId: z.string().uuid(),
relativePath: z.string().min(1),
line: z.number().int().positive().nullable().optional(),
column: z.number().int().positive().nullable().optional(),
displayPath: z.string().min(1),
});
export const workspaceFileResourceQuerySchema = z.object({
path: z.string().min(1),
workspace: workspaceFileSelectorSchema.optional(),
});
export const resolvedWorkspaceResourceSchema = z.object({
kind: workspaceFileResourceKindSchema,
provider: z.string().min(1),
title: z.string().min(1),
displayPath: z.string().min(1),
workspaceLabel: z.string().min(1),
workspaceKind: workspaceFileWorkspaceKindSchema,
workspaceId: z.string().uuid(),
contentType: z.string().nullable().optional(),
byteSize: z.number().int().nonnegative().nullable().optional(),
previewKind: workspaceFilePreviewKindSchema,
denialReason: z.string().nullable().optional(),
capabilities: z.object({
preview: z.boolean(),
download: z.literal(false),
listChildren: z.literal(false),
}),
});
export const workspaceFileContentSchema = z.object({
resource: resolvedWorkspaceResourceSchema,
content: z.object({
encoding: z.enum(["utf8", "base64"]),
data: z.string(),
}),
});
export type WorkspaceFileResourceQuery = z.infer<typeof workspaceFileResourceQuerySchema>;
+416
View File
@@ -0,0 +1,416 @@
import express from "express";
import type { Request } from "express";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import request from "supertest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { activityLog, companies, executionWorkspaces, goals, issues, projects, projectWorkspaces, type Db } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { errorHandler } from "../middleware/index.js";
import { createFileResourceLimiter, fileResourceRoutes, type WorkspaceFileResourceService } from "../routes/file-resources.js";
import {
WORKSPACE_FILE_TEXT_MAX_BYTES,
workspaceFileResourceService,
} from "../services/workspace-file-resources.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
const embeddedPostgresSupport = getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
type TestGraph = {
companyId: string;
otherCompanyId: string;
issueId: string;
otherIssueId: string;
projectId: string;
workspaceRoot: string;
executionRoot: string;
};
async function makeWorkspace() {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-file-resources-"));
const projectRoot = path.join(root, "project");
const executionRoot = path.join(root, "execution");
await fs.mkdir(projectRoot, { recursive: true });
await fs.mkdir(executionRoot, { recursive: true });
return { root, projectRoot, executionRoot };
}
async function seedGraph(db: Db, input: {
projectRoot: string;
executionRoot?: string | null;
projectSourceType?: string;
}): Promise<TestGraph> {
const suffix = crypto.randomUUID().slice(0, 8);
const companyId = crypto.randomUUID();
const otherCompanyId = crypto.randomUUID();
const goalId = crypto.randomUUID();
const otherGoalId = crypto.randomUUID();
const projectId = crypto.randomUUID();
const otherProjectId = crypto.randomUUID();
const projectWorkspaceId = crypto.randomUUID();
const otherProjectWorkspaceId = crypto.randomUUID();
const executionWorkspaceId = crypto.randomUUID();
const issueId = crypto.randomUUID();
const otherIssueId = crypto.randomUUID();
await db.insert(companies).values([
{ id: companyId, name: `Company ${suffix}`, issuePrefix: `F${suffix.slice(0, 4).toUpperCase()}` },
{ id: otherCompanyId, name: `Other ${suffix}`, issuePrefix: `G${suffix.slice(0, 4).toUpperCase()}` },
]);
await db.insert(goals).values([
{ id: goalId, companyId, title: "Goal", level: "company", status: "active" },
{ id: otherGoalId, companyId: otherCompanyId, title: "Other goal", level: "company", status: "active" },
]);
await db.insert(projects).values([
{ id: projectId, companyId, goalId, name: "Project", status: "in_progress" },
{ id: otherProjectId, companyId: otherCompanyId, goalId: otherGoalId, name: "Other project", status: "in_progress" },
]);
await db.insert(projectWorkspaces).values([
{
id: projectWorkspaceId,
companyId,
projectId,
name: "Primary workspace",
sourceType: input.projectSourceType ?? "local_path",
cwd: input.projectRoot,
isPrimary: true,
},
{
id: otherProjectWorkspaceId,
companyId: otherCompanyId,
projectId: otherProjectId,
name: "Other workspace",
sourceType: "local_path",
cwd: input.projectRoot,
isPrimary: true,
},
]);
await db.insert(issues).values([
{
id: issueId,
companyId,
projectId,
goalId,
projectWorkspaceId,
title: "Read a file",
status: "todo",
priority: "medium",
},
{
id: otherIssueId,
companyId: otherCompanyId,
projectId: otherProjectId,
goalId: otherGoalId,
projectWorkspaceId: otherProjectWorkspaceId,
title: "Other issue",
status: "todo",
priority: "medium",
},
]);
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
projectWorkspaceId,
sourceIssueId: issueId,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Issue worktree",
status: "active",
cwd: input.executionRoot ?? null,
providerType: input.executionRoot ? "git_worktree" : "remote_managed",
providerRef: input.executionRoot ?? "remote-workspace",
});
await db.update(issues).set({ executionWorkspaceId }).where(eq(issues.id, issueId));
return {
companyId,
otherCompanyId,
issueId,
otherIssueId,
projectId,
workspaceRoot: input.projectRoot,
executionRoot: input.executionRoot ?? input.projectRoot,
};
}
function createApp(db: Db, actor: Request["actor"]) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.actor = actor;
next();
});
app.use("/api", fileResourceRoutes(db));
app.use(errorHandler);
return app;
}
describeEmbeddedPostgres("workspace file resources", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let db: Db;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resources-");
db = tempDb.db;
}, 60_000);
afterAll(async () => {
await tempDb?.stop();
});
it("resolves and reads a project file without exposing absolute paths", async () => {
const { root, projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(projectRoot, "src", "app.ts"), "export const ok = true;\n", { encoding: "utf8" }).catch(async (error) => {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
await fs.mkdir(path.join(projectRoot, "src"), { recursive: true });
await fs.writeFile(path.join(projectRoot, "src", "app.ts"), "export const ok = true;\n", "utf8");
});
const app = createApp(db, {
type: "board",
userId: "board-user",
companyIds: [graph.companyId],
source: "session",
isInstanceAdmin: false,
});
const res = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content`)
.query({ workspace: "project", path: "src/app.ts" });
expect(res.status).toBe(200);
expect(res.headers["x-content-type-options"]).toBe("nosniff");
expect(res.body.resource.displayPath).toBe("src/app.ts");
expect(JSON.stringify(res.body)).not.toContain(root);
expect(res.body.content.data).toContain("export const ok");
});
it("falls back from an execution workspace miss to the project workspace", async () => {
const { projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(projectRoot, "README.md"), "# Project\n", "utf8");
const resolved = await workspaceFileResourceService(db).resolve(graph.issueId, {
path: "README.md",
workspace: "auto",
});
expect(resolved.workspaceKind).toBe("project_workspace");
expect(resolved.displayPath).toBe("README.md");
expect(resolved.capabilities.preview).toBe(true);
});
it("rejects traversal, encoded traversal, backslash traversal, and double-encoding without double-decoding", async () => {
const { projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(projectRoot, "safe.txt"), "safe\n", "utf8");
const app = createApp(db, {
type: "board",
userId: "board-user",
companyIds: [graph.companyId],
source: "session",
isInstanceAdmin: false,
});
expect((await request(app).get(`/api/issues/${graph.issueId}/file-resources/content?workspace=project&path=..%2Fsecret.txt`)).status).toBe(403);
expect((await request(app).get(`/api/issues/${graph.issueId}/file-resources/content?workspace=project&path=%2e%2e%2Fsecret.txt`)).status).toBe(403);
expect((await request(app).get(`/api/issues/${graph.issueId}/file-resources/content`).query({ workspace: "project", path: "..\\secret.txt" })).status).toBe(422);
const doubleEncoded = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content?workspace=project&path=%252e%252e%252Fsecret.txt`);
expect(doubleEncoded.status).toBe(404);
});
it("blocks symlink escapes and symlinks to denied sensitive files", async () => {
const { root, projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(root, "outside-secret.txt"), "secret\n", "utf8");
await fs.mkdir(path.join(projectRoot, "safe"), { recursive: true });
await fs.writeFile(path.join(projectRoot, "safe", ".env"), "TOKEN=secret\n", "utf8");
await fs.symlink(path.join(root, "outside-secret.txt"), path.join(projectRoot, "escape.txt"));
await fs.symlink(path.join(projectRoot, "safe", ".env"), path.join(projectRoot, "linked-env"));
const app = createApp(db, {
type: "board",
userId: "board-user",
companyIds: [graph.companyId],
source: "session",
isInstanceAdmin: false,
});
const escape = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content`)
.query({ workspace: "project", path: "escape.txt" });
const linkedSecret = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content`)
.query({ workspace: "project", path: "linked-env" });
expect(escape.status).toBe(403);
expect(linkedSecret.status).toBe(403);
});
it("rejects denied paths, non-regular files, oversized text, binary, and HTML while previewing SVG as source", async () => {
const { projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.mkdir(path.join(projectRoot, ".git"), { recursive: true });
await fs.writeFile(path.join(projectRoot, ".git", "config"), "[core]\n", "utf8");
await fs.mkdir(path.join(projectRoot, "folder"), { recursive: true });
await fs.writeFile(path.join(projectRoot, "big.txt"), Buffer.alloc(WORKSPACE_FILE_TEXT_MAX_BYTES + 1, "a"));
await fs.writeFile(path.join(projectRoot, "blob.bin"), Buffer.from([0, 1, 2, 3]));
await fs.writeFile(path.join(projectRoot, "index.html"), "<script>alert(1)</script>", "utf8");
await fs.writeFile(path.join(projectRoot, "icon.svg"), "<svg></svg>\n", "utf8");
const app = createApp(db, {
type: "board",
userId: "board-user",
companyIds: [graph.companyId],
source: "session",
isInstanceAdmin: false,
});
for (const filePath of [".git/config", "folder", "big.txt", "blob.bin", "index.html"]) {
const res = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content`)
.query({ workspace: "project", path: filePath });
expect([403, 422]).toContain(res.status);
}
const svg = await request(app)
.get(`/api/issues/${graph.issueId}/file-resources/content`)
.query({ workspace: "project", path: "icon.svg" });
expect(svg.status).toBe(200);
expect(svg.body.resource.previewKind).toBe("text");
expect(svg.body.content.data).toContain("<svg");
});
it("rejects remote workspaces without fetching provider resources", async () => {
const { projectRoot } = await makeWorkspace();
const graph = await seedGraph(db, {
projectRoot,
executionRoot: null,
projectSourceType: "remote_managed",
});
const resolved = await workspaceFileResourceService(db).resolve(graph.issueId, {
path: "README.md",
workspace: "project",
});
expect(resolved.kind).toBe("remote_resource");
expect(resolved.capabilities.preview).toBe(false);
await expect(workspaceFileResourceService(db).readContent(graph.issueId, {
path: "http://169.254.169.254/latest/meta-data/",
workspace: "project",
})).rejects.toMatchObject({ status: 422 });
});
it("blocks agents and cross-company board users before content reads", async () => {
const { projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(projectRoot, "README.md"), "# Secret\n", "utf8");
const agentApp = createApp(db, {
type: "agent",
agentId: crypto.randomUUID(),
companyId: graph.companyId,
source: "agent_key",
});
const boardApp = createApp(db, {
type: "board",
userId: "mallory",
companyIds: [graph.otherCompanyId],
source: "session",
isInstanceAdmin: false,
});
expect((await request(agentApp).get(`/api/issues/${graph.issueId}/file-resources/content`).query({ path: "README.md" })).status).toBe(403);
expect((await request(boardApp).get(`/api/issues/${graph.issueId}/file-resources/content`).query({ path: "README.md" })).status).toBe(403);
});
it("logs successful content reads and denied security-relevant attempts", async () => {
const { projectRoot, executionRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot });
await fs.writeFile(path.join(projectRoot, "README.md"), "# Project\n", "utf8");
await fs.writeFile(path.join(projectRoot, ".env"), "TOKEN=secret\n", "utf8");
const app = createApp(db, {
type: "board",
userId: "board-user",
companyIds: [graph.companyId],
source: "session",
isInstanceAdmin: false,
});
await request(app).get(`/api/issues/${graph.issueId}/file-resources/content`).query({ workspace: "project", path: "README.md" });
await request(app).get(`/api/issues/${graph.issueId}/file-resources/content`).query({ workspace: "project", path: ".env" });
const rows = await db.select().from(activityLog).where(eq(activityLog.entityId, graph.issueId));
expect(rows.some((row) => row.action === "issue.file_resource_content_read")).toBe(true);
expect(rows.some((row) => row.action === "issue.file_resource_content_denied")).toBe(true);
expect(JSON.stringify(rows.map((row) => row.details))).not.toContain(projectRoot);
});
});
describe("file resource route guards", () => {
it("enforces bounded rate and concurrency limits", async () => {
let releaseSlowRead: (() => void) | null = null;
let slowReadStarted: (() => void) | null = null;
const slowRead = new Promise<void>((resolve) => {
releaseSlowRead = resolve;
});
const readStarted = new Promise<void>((resolve) => {
slowReadStarted = resolve;
});
const service: WorkspaceFileResourceService = {
getIssue: vi.fn(async () => ({ companyId: "company-1" })),
resolve: vi.fn(async () => {
slowReadStarted?.();
await slowRead;
return {
kind: "file",
provider: "local_fs",
title: "README.md",
displayPath: "README.md",
workspaceLabel: "Workspace",
workspaceKind: "project_workspace",
workspaceId: "11111111-1111-4111-8111-111111111111",
previewKind: "text",
capabilities: { preview: true, download: false, listChildren: false },
};
}),
readContent: vi.fn(async () => {
throw new Error("not used");
}),
};
const app = express();
app.use((req, _res, next) => {
req.actor = {
type: "board",
userId: "board-user",
companyIds: ["company-1"],
source: "session",
isInstanceAdmin: false,
};
next();
});
app.use("/api", fileResourceRoutes({} as Db, {
service,
limiter: createFileResourceLimiter({ maxConcurrent: 1, maxRequests: 2, windowMs: 60_000 }),
}));
app.use(errorHandler);
const first = request(app).get("/api/issues/issue-1/file-resources/resolve").query({ path: "README.md" });
const firstResponse = first.then((res) => res);
await readStarted;
const second = await request(app).get("/api/issues/issue-1/file-resources/resolve").query({ path: "README.md" });
expect(second.status).toBe(429);
releaseSlowRead?.();
expect((await firstResponse).status).toBe(200);
const third = await request(app).get("/api/issues/issue-1/file-resources/resolve").query({ path: "README.md" });
expect(third.status).toBe(429);
});
});
+2
View File
@@ -17,6 +17,7 @@ import { agentRoutes } from "./routes/agents.js";
import { projectRoutes } from "./routes/projects.js";
import { issueRoutes } from "./routes/issues.js";
import { issueTreeControlRoutes } from "./routes/issue-tree-control.js";
import { fileResourceRoutes } from "./routes/file-resources.js";
import { routineRoutes } from "./routes/routines.js";
import { environmentRoutes } from "./routes/environments.js";
import { executionWorkspaceRoutes } from "./routes/execution-workspaces.js";
@@ -222,6 +223,7 @@ export async function createApp(
pluginWorkerManager: workerManager,
}));
api.use(issueTreeControlRoutes(db));
api.use(fileResourceRoutes(db));
api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
api.use(environmentRoutes(db, { pluginWorkerManager: workerManager }));
api.use(executionWorkspaceRoutes(db));
+209
View File
@@ -0,0 +1,209 @@
import { Router } from "express";
import type { Db } from "@paperclipai/db";
import {
workspaceFileResourceQuerySchema,
type ResolvedWorkspaceResource,
type WorkspaceFileContent,
} from "@paperclipai/shared";
import { HttpError, unprocessable } from "../errors.js";
import { workspaceFileResourceService } from "../services/index.js";
import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
import { logActivity } from "../services/activity-log.js";
export type WorkspaceFileResourceService = {
getIssue(issueId: string): Promise<{ companyId: string }>;
resolve(issueId: string, input: { path: string; workspace?: "auto" | "execution" | "project" | null }): Promise<ResolvedWorkspaceResource>;
readContent(issueId: string, input: { path: string; workspace?: "auto" | "execution" | "project" | null }): Promise<WorkspaceFileContent>;
};
type FileResourceLimiter = {
acquire(key: string): () => void;
};
export function createFileResourceLimiter(opts: {
maxConcurrent?: number;
maxRequests?: number;
windowMs?: number;
} = {}): FileResourceLimiter {
const maxConcurrent = opts.maxConcurrent ?? 6;
const maxRequests = opts.maxRequests ?? 120;
const windowMs = opts.windowMs ?? 60_000;
const activeByKey = new Map<string, number>();
const windowsByKey = new Map<string, { startedAt: number; count: number }>();
return {
acquire(key: string) {
const now = Date.now();
const window = windowsByKey.get(key);
if (!window || now - window.startedAt >= windowMs) {
windowsByKey.set(key, { startedAt: now, count: 1 });
} else {
window.count += 1;
if (window.count > maxRequests) {
throw new HttpError(429, "Too many file preview requests");
}
}
const active = activeByKey.get(key) ?? 0;
if (active >= maxConcurrent) {
throw new HttpError(429, "Too many concurrent file preview requests");
}
activeByKey.set(key, active + 1);
return () => {
const current = activeByKey.get(key) ?? 0;
if (current <= 1) activeByKey.delete(key);
else activeByKey.set(key, current - 1);
};
},
};
}
function limiterKey(companyId: string, actorId: string, issueId: string) {
return `${companyId}:${actorId}:${issueId}`;
}
function readQuery(query: unknown) {
const parsed = workspaceFileResourceQuerySchema.parse(query);
return {
path: parsed.path,
workspace: parsed.workspace ?? "auto",
};
}
function activityDetails(input: {
outcome: "success" | "denied";
workspaceKind?: string | null;
workspaceId?: string | null;
displayPath?: string | null;
denialReason?: string | null;
byteSize?: number | null;
contentType?: string | null;
}) {
return {
outcome: input.outcome,
...(input.workspaceKind ? { workspaceKind: input.workspaceKind } : {}),
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
...(input.displayPath ? { displayPath: input.displayPath } : {}),
...(input.denialReason ? { denialReason: input.denialReason } : {}),
...(typeof input.byteSize === "number" ? { byteSize: input.byteSize } : {}),
...(input.contentType ? { contentType: input.contentType } : {}),
};
}
function denialReasonFromError(error: unknown) {
if (!(error instanceof HttpError)) return "unknown";
const details = error.details;
if (details && typeof details === "object" && "code" in details) {
const code = (details as { code?: unknown }).code;
if (typeof code === "string" && code.length > 0) return code;
}
return error.message;
}
export function fileResourceRoutes(db: Db, opts: {
service?: WorkspaceFileResourceService;
limiter?: FileResourceLimiter;
} = {}) {
const router = Router();
const svc = opts.service ?? workspaceFileResourceService(db);
const limiter = opts.limiter ?? createFileResourceLimiter();
async function logDeniedAttempt(input: {
companyId: string;
actor: ReturnType<typeof getActorInfo>;
issueId: string;
displayPath: string;
error: unknown;
}) {
await logActivity(db, {
companyId: input.companyId,
actorType: input.actor.actorType,
actorId: input.actor.actorId,
action: "issue.file_resource_content_denied",
entityType: "issue",
entityId: input.issueId,
agentId: input.actor.agentId,
runId: input.actor.runId,
details: activityDetails({
outcome: "denied",
displayPath: input.displayPath,
denialReason: denialReasonFromError(input.error),
}),
});
}
router.get("/issues/:issueId/file-resources/resolve", async (req, res) => {
assertBoard(req);
const issue = await svc.getIssue(req.params.issueId);
assertCompanyAccess(req, issue.companyId);
const actor = getActorInfo(req);
const query = readQuery(req.query);
const release = limiter.acquire(limiterKey(issue.companyId, actor.actorId, req.params.issueId));
try {
const result = await svc.resolve(req.params.issueId, query);
res.json(result);
} catch (error) {
await logDeniedAttempt({
companyId: issue.companyId,
actor,
issueId: req.params.issueId,
displayPath: query.path,
error,
});
throw error;
} finally {
release();
}
});
router.get("/issues/:issueId/file-resources/content", async (req, res) => {
assertBoard(req);
const issue = await svc.getIssue(req.params.issueId);
assertCompanyAccess(req, issue.companyId);
const actor = getActorInfo(req);
const query = readQuery(req.query);
const release = limiter.acquire(limiterKey(issue.companyId, actor.actorId, req.params.issueId));
try {
let result: WorkspaceFileContent | null = null;
try {
result = await svc.readContent(req.params.issueId, query);
} catch (error) {
await logDeniedAttempt({
companyId: issue.companyId,
actor,
issueId: req.params.issueId,
displayPath: query.path,
error,
});
throw error;
}
if (!result) throw unprocessable("Workspace file cannot be previewed");
await logActivity(db, {
companyId: issue.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
action: "issue.file_resource_content_read",
entityType: "issue",
entityId: req.params.issueId,
agentId: actor.agentId,
runId: actor.runId,
details: activityDetails({
outcome: "success",
workspaceKind: result.resource.workspaceKind,
workspaceId: result.resource.workspaceId,
displayPath: result.resource.displayPath,
byteSize: result.resource.byteSize ?? null,
contentType: result.resource.contentType ?? null,
}),
});
res.set("X-Content-Type-Options", "nosniff");
res.json(result);
} finally {
release();
}
});
return router;
}
+1
View File
@@ -6,6 +6,7 @@ export { agentRoutes } from "./agents.js";
export { projectRoutes } from "./projects.js";
export { issueRoutes } from "./issues.js";
export { issueTreeControlRoutes } from "./issue-tree-control.js";
export { fileResourceRoutes, createFileResourceLimiter } from "./file-resources.js";
export { routineRoutes } from "./routines.js";
export { goalRoutes } from "./goals.js";
export { approvalRoutes } from "./approvals.js";
+1
View File
@@ -68,6 +68,7 @@ export { teamsCatalogService } from "./teams-catalog.js";
export { environmentService } from "./environments.js";
export { executionWorkspaceService } from "./execution-workspaces.js";
export { workspaceOperationService } from "./workspace-operations.js";
export { workspaceFileResourceService } from "./workspace-file-resources.js";
export { workProductService } from "./work-products.js";
export { logActivity, type LogActivityInput } from "./activity-log.js";
export { notifyHireApproved, type NotifyHireApprovedInput } from "./hire-hook.js";
@@ -0,0 +1,485 @@
import fs from "node:fs/promises";
import path from "node:path";
import { and, desc, eq, inArray, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { executionWorkspaces, issues, projectWorkspaces } from "@paperclipai/db";
import type {
ResolvedWorkspaceResource,
WorkspaceFileContent,
WorkspaceFilePreviewKind,
WorkspaceFileSelector,
WorkspaceFileWorkspaceKind,
} from "@paperclipai/shared";
import { HttpError, notFound, unprocessable } from "../errors.js";
export const WORKSPACE_FILE_TEXT_MAX_BYTES = 512 * 1024;
export const WORKSPACE_FILE_MEDIA_MAX_BYTES = 10 * 1024 * 1024;
const MAX_RELATIVE_PATH_BYTES = 4096;
const TEXT_SNIFF_BYTES = 4096;
const DENIED_SEGMENTS = new Set([
".git",
".paperclip",
"node_modules",
".pnpm-store",
".yarn",
".cache",
".turbo",
".next",
".vite",
".vercel",
"dist",
"build",
"coverage",
"runtime-services",
".runtime",
]);
const TEXT_EXTENSIONS = new Set([
".c",
".cc",
".conf",
".cpp",
".css",
".csv",
".go",
".h",
".html",
".htm",
".java",
".js",
".json",
".jsx",
".log",
".md",
".mjs",
".py",
".rb",
".rs",
".sh",
".sql",
".svg",
".toml",
".ts",
".tsx",
".txt",
".xml",
".yaml",
".yml",
]);
const IMAGE_CONTENT_TYPES = new Map([
[".gif", "image/gif"],
[".jpeg", "image/jpeg"],
[".jpg", "image/jpeg"],
[".png", "image/png"],
[".webp", "image/webp"],
]);
type IssueRow = typeof issues.$inferSelect;
type ExecutionWorkspaceRow = typeof executionWorkspaces.$inferSelect;
type ProjectWorkspaceRow = typeof projectWorkspaces.$inferSelect;
type WorkspaceCandidate = {
workspaceKind: WorkspaceFileWorkspaceKind;
workspaceId: string;
provider: string;
label: string;
rootPath: string | null;
remote: boolean;
};
type NormalizedPath = {
relativePath: string;
segments: string[];
};
type LocalResolvedFile = {
resource: ResolvedWorkspaceResource;
realPath: string;
};
function previewCapForKind(kind: WorkspaceFilePreviewKind) {
return kind === "image" || kind === "pdf" ? WORKSPACE_FILE_MEDIA_MAX_BYTES : WORKSPACE_FILE_TEXT_MAX_BYTES;
}
function relativePathFromReal(rootReal: string, targetReal: string) {
return path.relative(rootReal, targetReal).split(path.sep).join(path.posix.sep);
}
function normalizeWorkspaceRelativePath(input: string): NormalizedPath {
const trimmed = input.trim();
if (!trimmed) throw unprocessable("Workspace file path is required", { code: "invalid_path" });
if (Buffer.byteLength(trimmed, "utf8") > MAX_RELATIVE_PATH_BYTES) {
throw unprocessable("Workspace file path is too long", { code: "invalid_path" });
}
if (trimmed.includes("\0")) throw unprocessable("Workspace file path contains an invalid character", { code: "invalid_path" });
if (/^file:\/\//i.test(trimmed)) throw unprocessable("File URLs are not supported", { code: "invalid_path" });
if (/^[a-zA-Z]:/.test(trimmed)) throw unprocessable("Windows drive paths are not supported", { code: "invalid_path" });
if (trimmed.includes("\\")) throw unprocessable("Workspace file paths must use forward slashes", { code: "invalid_path" });
if (path.posix.isAbsolute(trimmed)) throw unprocessable("Workspace file path must be relative", { code: "invalid_path" });
const normalized = path.posix.normalize(trimmed);
if (normalized === "." || normalized === ".." || normalized.startsWith("../")) {
throw new HttpError(403, "Workspace file path is outside the workspace", { code: "outside_workspace" });
}
return {
relativePath: normalized,
segments: normalized.split("/").filter(Boolean),
};
}
function denyReasonForPathSegments(segments: string[]): string | null {
const lowerSegments = segments.map((segment) => segment.toLowerCase());
if (lowerSegments.some((segment) => DENIED_SEGMENTS.has(segment))) return "denied_path_segment";
const fileName = lowerSegments.at(-1) ?? "";
if (fileName === ".env" || fileName.startsWith(".env.")) return "denied_secret";
if (fileName.endsWith(".pem") || fileName.endsWith(".key") || fileName.endsWith(".p12") || fileName.endsWith(".pfx")) {
return "denied_secret";
}
if (["id_rsa", "id_ed25519", ".npmrc", ".pypirc", ".netrc", "kubeconfig"].includes(fileName)) return "denied_secret";
if (lowerSegments.includes(".aws") || lowerSegments.includes(".ssh")) return "denied_secret";
if (lowerSegments.length >= 2 && lowerSegments.at(-2) === ".docker" && fileName === "config.json") return "denied_secret";
if (lowerSegments.length >= 2 && lowerSegments.at(-2) === ".kube" && fileName === "config") return "denied_secret";
return null;
}
function throwIfDenied(segments: string[]) {
const denialReason = denyReasonForPathSegments(segments);
if (denialReason) {
throw new HttpError(403, "Workspace file path is denied by policy", { code: denialReason });
}
}
function contentTypeForPath(filePath: string): string | null {
const ext = path.extname(filePath).toLowerCase();
if (IMAGE_CONTENT_TYPES.has(ext)) return IMAGE_CONTENT_TYPES.get(ext) ?? null;
if (ext === ".pdf") return "application/pdf";
if (ext === ".svg") return "image/svg+xml";
if (ext === ".html" || ext === ".htm") return "text/html";
if (TEXT_EXTENSIONS.has(ext)) return "text/plain; charset=utf-8";
return null;
}
function previewKindForKnownContentType(contentType: string | null): WorkspaceFilePreviewKind | null {
if (!contentType) return null;
if (contentType.startsWith("image/") && contentType !== "image/svg+xml") return "image";
if (contentType === "application/pdf") return "pdf";
if (contentType === "text/html") return "unsupported";
if (contentType === "image/svg+xml" || contentType.startsWith("text/")) return "text";
return "unsupported";
}
function looksLikeText(buffer: Buffer) {
if (buffer.length === 0) return true;
let controlBytes = 0;
for (const byte of buffer) {
if (byte === 0) return false;
if (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13) controlBytes += 1;
}
return controlBytes / buffer.length < 0.02;
}
async function sniffUnknownPreviewKind(realPath: string): Promise<WorkspaceFilePreviewKind> {
const handle = await fs.open(realPath, "r");
try {
const buffer = Buffer.alloc(TEXT_SNIFF_BYTES);
const { bytesRead } = await handle.read(buffer, 0, TEXT_SNIFF_BYTES, 0);
return looksLikeText(buffer.subarray(0, bytesRead)) ? "text" : "unsupported";
} finally {
await handle.close();
}
}
function remoteResource(candidate: WorkspaceCandidate, relativePath: string): ResolvedWorkspaceResource {
return {
kind: "remote_resource",
provider: candidate.provider || "remote_managed",
title: path.posix.basename(relativePath),
displayPath: relativePath,
workspaceLabel: candidate.label,
workspaceKind: candidate.workspaceKind,
workspaceId: candidate.workspaceId,
contentType: null,
byteSize: null,
previewKind: "unsupported",
denialReason: "remote_workspace",
capabilities: {
preview: false,
download: false,
listChildren: false,
},
};
}
function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceCandidate {
const provider = row.providerType || row.strategyType || "local_fs";
const rootPath = row.cwd || row.providerRef || null;
const remote = !["local_fs", "git_worktree"].includes(provider) || !rootPath || row.status !== "active" || row.closedAt != null;
return {
workspaceKind: "execution_workspace",
workspaceId: row.id,
provider,
label: row.name,
rootPath,
remote,
};
}
function candidateFromProjectWorkspace(row: ProjectWorkspaceRow): WorkspaceCandidate {
const provider = row.sourceType === "git_worktree" ? "git_worktree" : row.sourceType === "local_path" ? "local_fs" : row.sourceType;
const rootPath = row.cwd ?? null;
const remote = !["local_fs", "git_worktree"].includes(provider) || !rootPath;
return {
workspaceKind: "project_workspace",
workspaceId: row.id,
provider,
label: row.name,
rootPath,
remote,
};
}
async function statLocalCandidate(candidate: WorkspaceCandidate, normalized: NormalizedPath): Promise<LocalResolvedFile> {
if (!candidate.rootPath) {
throw unprocessable("Workspace is not locally readable", { code: "remote_workspace" });
}
throwIfDenied(normalized.segments);
let rootReal: string;
try {
rootReal = await fs.realpath(candidate.rootPath);
} catch {
throw unprocessable("Workspace is not available on this host", { code: "workspace_unavailable" });
}
const targetLexical = path.resolve(rootReal, ...normalized.segments);
let targetReal: string;
try {
targetReal = await fs.realpath(targetLexical);
} catch {
throw notFound("Workspace file not found");
}
const realRelative = path.relative(rootReal, targetReal);
if (realRelative === "" || realRelative.startsWith("..") || path.isAbsolute(realRelative)) {
throw new HttpError(403, "Workspace file path is outside the workspace", { code: "outside_workspace" });
}
throwIfDenied(relativePathFromReal(rootReal, targetReal).split("/").filter(Boolean));
const stat = await fs.stat(targetReal);
if (!stat.isFile()) {
throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
}
const contentType = contentTypeForPath(normalized.relativePath);
let previewKind = previewKindForKnownContentType(contentType);
if (!previewKind) {
previewKind = await sniffUnknownPreviewKind(targetReal);
}
const cap = previewCapForKind(previewKind);
const tooLarge = stat.size > cap;
const unsupported = previewKind === "unsupported";
const denialReason = tooLarge ? "too_large" : unsupported ? "unsupported_content" : null;
return {
realPath: targetReal,
resource: {
kind: "file",
provider: candidate.provider,
title: path.posix.basename(normalized.relativePath),
displayPath: normalized.relativePath,
workspaceLabel: candidate.label,
workspaceKind: candidate.workspaceKind,
workspaceId: candidate.workspaceId,
contentType: contentType ?? (previewKind === "text" ? "text/plain; charset=utf-8" : "application/octet-stream"),
byteSize: stat.size,
previewKind,
denialReason,
capabilities: {
preview: !tooLarge && !unsupported,
download: false,
listChildren: false,
},
},
};
}
async function readStableFile(realPath: string, maxBytes: number) {
const handle = await fs.open(realPath, "r");
try {
const before = await handle.stat();
if (!before.isFile()) throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
if (before.size > maxBytes) throw unprocessable("Workspace file is too large to preview", { code: "too_large" });
const data = await handle.readFile();
const after = await handle.stat();
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || after.mtimeMs !== before.mtimeMs) {
throw unprocessable("Workspace file changed while being read", { code: "file_changed" });
}
return data;
} finally {
await handle.close();
}
}
export function workspaceFileResourceService(db: Db) {
async function getIssue(issueId: string): Promise<IssueRow> {
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)).limit(1);
if (!issue) throw notFound("Issue not found");
return issue;
}
async function listCandidates(issue: IssueRow, selector: WorkspaceFileSelector): Promise<WorkspaceCandidate[]> {
const candidates: WorkspaceCandidate[] = [];
if ((selector === "auto" || selector === "execution") && issue.projectId) {
const executionIds = [issue.executionWorkspaceId].filter((id): id is string => Boolean(id));
let executionRows: ExecutionWorkspaceRow[] = [];
if (executionIds.length > 0) {
executionRows = await db.select().from(executionWorkspaces).where(
and(
eq(executionWorkspaces.companyId, issue.companyId),
inArray(executionWorkspaces.id, executionIds),
),
);
}
const sourceIssueIds = [issue.id, issue.parentId].filter((id): id is string => Boolean(id));
if (sourceIssueIds.length > 0) {
const activeRows = await db.select().from(executionWorkspaces).where(
and(
eq(executionWorkspaces.companyId, issue.companyId),
eq(executionWorkspaces.projectId, issue.projectId),
inArray(executionWorkspaces.sourceIssueId, sourceIssueIds),
eq(executionWorkspaces.status, "active"),
isNull(executionWorkspaces.closedAt),
),
).orderBy(desc(executionWorkspaces.lastUsedAt)).limit(2);
executionRows = [...executionRows, ...activeRows];
}
const seen = new Set<string>();
for (const row of executionRows) {
if (seen.has(row.id)) continue;
seen.add(row.id);
candidates.push(candidateFromExecutionWorkspace(row));
}
}
if ((selector === "auto" || selector === "project") && issue.projectId) {
if (issue.projectWorkspaceId) {
const rows = await db.select().from(projectWorkspaces).where(
and(
eq(projectWorkspaces.companyId, issue.companyId),
eq(projectWorkspaces.projectId, issue.projectId),
eq(projectWorkspaces.id, issue.projectWorkspaceId),
),
).limit(1);
if (rows[0]) candidates.push(candidateFromProjectWorkspace(rows[0]));
}
const primaryRows = await db.select().from(projectWorkspaces).where(
and(
eq(projectWorkspaces.companyId, issue.companyId),
eq(projectWorkspaces.projectId, issue.projectId),
eq(projectWorkspaces.isPrimary, true),
),
).limit(1);
if (primaryRows[0] && !candidates.some((candidate) => candidate.workspaceId === primaryRows[0]!.id)) {
candidates.push(candidateFromProjectWorkspace(primaryRows[0]));
}
}
return candidates;
}
async function resolve(issueId: string, input: {
path: string;
workspace?: WorkspaceFileSelector | null;
}): Promise<ResolvedWorkspaceResource> {
const issue = await getIssue(issueId);
const selector = input.workspace ?? "auto";
const normalized = normalizeWorkspaceRelativePath(input.path);
const candidates = await listCandidates(issue, selector);
if (candidates.length === 0) {
throw unprocessable("No workspace is available for this issue", { code: "no_workspace" });
}
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
if (selector !== "auto") return remoteResource(candidate, normalized.relativePath);
continue;
}
try {
return (await statLocalCandidate(candidate, normalized)).resource;
} catch (error) {
if (error instanceof Error && "status" in error && (error as { status?: number }).status === 404 && selector === "auto") {
lastNotFound = error;
continue;
}
throw error;
}
}
if (lastNotFound) throw lastNotFound;
throw unprocessable("No local-readable workspace is available for this issue", { code: "no_local_workspace" });
}
async function readContent(issueId: string, input: {
path: string;
workspace?: WorkspaceFileSelector | null;
}): Promise<WorkspaceFileContent> {
const issue = await getIssue(issueId);
const selector = input.workspace ?? "auto";
const normalized = normalizeWorkspaceRelativePath(input.path);
const candidates = await listCandidates(issue, selector);
if (candidates.length === 0) {
throw unprocessable("No workspace is available for this issue", { code: "no_workspace" });
}
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
if (selector !== "auto") throw unprocessable("Remote workspaces cannot be previewed by the server", { code: "remote_workspace" });
continue;
}
let resolved: LocalResolvedFile;
try {
resolved = await statLocalCandidate(candidate, normalized);
} catch (error) {
if (error instanceof Error && "status" in error && (error as { status?: number }).status === 404 && selector === "auto") {
lastNotFound = error;
continue;
}
throw error;
}
if (!resolved.resource.capabilities.preview) {
throw unprocessable("Workspace file cannot be previewed", { code: resolved.resource.denialReason ?? "unsupported_content" });
}
const cap = previewCapForKind(resolved.resource.previewKind);
const data = await readStableFile(resolved.realPath, cap);
if (resolved.resource.previewKind === "text" && !looksLikeText(data.subarray(0, Math.min(data.length, TEXT_SNIFF_BYTES)))) {
throw unprocessable("Workspace file is not a text file", { code: "binary_content" });
}
return {
resource: resolved.resource,
content: {
encoding: resolved.resource.previewKind === "text" ? "utf8" : "base64",
data: resolved.resource.previewKind === "text" ? data.toString("utf8") : data.toString("base64"),
},
};
}
if (lastNotFound) throw lastNotFound;
throw unprocessable("No local-readable workspace is available for this issue", { code: "no_local_workspace" });
}
return {
getIssue,
resolve,
readContent,
};
}