From 2f302334f7ea754bf98f16bde545b66f17a1e041 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 11 Jul 2026 11:35:27 +0000 Subject: [PATCH] feat(server): stream issue attachment archives Co-Authored-By: Paperclip --- server/package.json | 2 + .../__tests__/issue-attachment-routes.test.ts | 178 +++++++++++++++++- server/src/issue-attachment-archive.ts | 88 +++++++++ server/src/routes/issues.ts | 100 ++++++++++ 4 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 server/src/issue-attachment-archive.ts diff --git a/server/package.json b/server/package.json index 8410e8e088e..ca28582ff06 100644 --- a/server/package.json +++ b/server/package.json @@ -79,6 +79,7 @@ "sharp": "^0.35.3", "ssh2": "^1.17.0", "ws": "^8.21.1", + "zip-stream": "^7.0.5", "zod": "^3.24.2" }, "devDependencies": { @@ -90,6 +91,7 @@ "@types/sharp": "^0.32.0", "@types/supertest": "^6.0.2", "@types/ws": "^8.18.1", + "@types/zip-stream": "^7.0.0", "cross-env": "^10.1.0", "supertest": "^7.0.0", "tsx": "^4.23.1", diff --git a/server/src/__tests__/issue-attachment-routes.test.ts b/server/src/__tests__/issue-attachment-routes.test.ts index 2103aa58809..0b3c51b610f 100644 --- a/server/src/__tests__/issue-attachment-routes.test.ts +++ b/server/src/__tests__/issue-attachment-routes.test.ts @@ -1,5 +1,6 @@ import { Readable } from "node:stream"; import type { IncomingMessage } from "node:http"; +import { inflateRawSync } from "node:zlib"; import express from "express"; import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -10,6 +11,10 @@ const mockIssueService = vi.hoisted(() => ({ getByIdentifier: vi.fn(), createAttachment: vi.fn(), getAttachmentById: vi.fn(), + listAttachments: vi.fn(), +})); +const mockDocumentService = vi.hoisted(() => ({ + listIssueDocuments: vi.fn(), })); const mockCompanyService = vi.hoisted(() => ({ getById: vi.fn(), @@ -56,7 +61,7 @@ function registerRouteMocks() { companySkillService: () => ({}), companyService: () => mockCompanyService, documentAnnotationService: () => ({ remapOpenThreadsForDocument: async () => [] }), - documentService: () => ({}), + documentService: () => mockDocumentService, executionWorkspaceService: () => ({}), feedbackService: () => ({ listIssueVotesForUser: vi.fn(async () => []), @@ -204,6 +209,33 @@ function parseBinaryResponse(res: IncomingMessage, callback: (error: Error | nul res.on("error", callback); } +function readZipEntries(buffer: Buffer): Map { + const endOfCentralDirectory = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + if (endOfCentralDirectory < 0) throw new Error("ZIP end-of-central-directory record not found"); + const entryCount = buffer.readUInt16LE(endOfCentralDirectory + 10); + let centralOffset = buffer.readUInt32LE(endOfCentralDirectory + 16); + const entries = new Map(); + + for (let index = 0; index < entryCount; index += 1) { + if (buffer.readUInt32LE(centralOffset) !== 0x02014b50) throw new Error("Invalid ZIP central directory entry"); + const compressionMethod = buffer.readUInt16LE(centralOffset + 10); + const compressedSize = buffer.readUInt32LE(centralOffset + 20); + const filenameLength = buffer.readUInt16LE(centralOffset + 28); + const extraLength = buffer.readUInt16LE(centralOffset + 30); + const commentLength = buffer.readUInt16LE(centralOffset + 32); + const localOffset = buffer.readUInt32LE(centralOffset + 42); + const filename = buffer.subarray(centralOffset + 46, centralOffset + 46 + filenameLength).toString("utf8"); + const localFilenameLength = buffer.readUInt16LE(localOffset + 26); + const localExtraLength = buffer.readUInt16LE(localOffset + 28); + const dataOffset = localOffset + 30 + localFilenameLength + localExtraLength; + const compressed = buffer.subarray(dataOffset, dataOffset + compressedSize); + entries.set(filename, compressionMethod === 0 ? compressed : inflateRawSync(compressed)); + centralOffset += 46 + filenameLength + extraLength + commentLength; + } + + return entries; +} + describe("normalizeIssueAttachmentMaxBytes", () => { it("keeps the process-level attachment cap as the final cap", async () => { const previous = process.env.PAPERCLIP_ATTACHMENT_MAX_BYTES; @@ -253,6 +285,8 @@ describe("issue attachment routes", () => { assigneeUserId: null, identifier: "PAP-1", }); + mockIssueService.listAttachments.mockResolvedValue([]); + mockDocumentService.listIssueDocuments.mockResolvedValue([]); mockCompanyService.getById.mockResolvedValue({ id: "company-1", attachmentMaxBytes: 1024 * 1024 * 1024, @@ -262,6 +296,148 @@ describe("issue attachment routes", () => { mockWorkProductService.update.mockReset(); }); + it("streams attachments and visible documents in a private ZIP archive", async () => { + const storage = createStorageService(); + const firstAttachment = { + ...makeAttachment("application/octet-stream", "report.txt"), + id: "attachment-a", + objectKey: "issues/issue-1/report-a.txt", + issueCommentId: "comment-1", + }; + const duplicateAttachment = { + ...makeAttachment("application/octet-stream", "report.txt"), + id: "attachment-b", + objectKey: "issues/issue-1/report-b.txt", + }; + const unnamedAttachment = { + ...makeAttachment("image/png", "ignored.png"), + id: "attachment-c", + objectKey: "issues/issue-1/unnamed", + originalFilename: null, + }; + mockIssueService.listAttachments.mockResolvedValue([ + duplicateAttachment, + unnamedAttachment, + firstAttachment, + ]); + mockDocumentService.listIssueDocuments.mockImplementation(async (_issueId, options) => options?.includeSystem + ? [ + { key: "plan", body: "# Plan\n", updatedAt: new Date("2026-01-02T00:00:00.000Z") }, + { key: "continuation-summary", body: "internal", updatedAt: new Date("2026-01-02T00:00:00.000Z") }, + ] + : [{ key: "plan", body: "# Plan\n", updatedAt: new Date("2026-01-02T00:00:00.000Z") }]); + vi.mocked(storage.getObject).mockImplementation(async (_companyId, objectKey) => ({ + stream: Readable.from({ + "issues/issue-1/report-a.txt": Buffer.from([0x00, 0x01, 0xff]), + "issues/issue-1/report-b.txt": Buffer.from("second"), + "issues/issue-1/unnamed": Buffer.from([0x89, 0x50, 0x4e, 0x47]), + }[objectKey] ?? Buffer.alloc(0)), + contentLength: 0, + })); + + const app = await createApp(storage); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive") + .buffer(true) + .parse(parseBinaryResponse); + + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toContain("application/zip"); + expect(res.headers["content-disposition"]).toBe('attachment; filename="attachments-PAP-1.zip"'); + expect(res.headers["cache-control"]).toBe("private, no-store, no-transform"); + expect(res.headers["x-content-type-options"]).toBe("nosniff"); + expect(res.headers["content-length"]).toBeUndefined(); + expect(res.headers["transfer-encoding"]).toBe("chunked"); + + const entries = readZipEntries(Buffer.from(res.body)); + expect([...entries.keys()].sort()).toEqual([ + "attachments/attachment-attachment-c.png", + "attachments/report-2.txt", + "attachments/report.txt", + "documents/plan.md", + ]); + expect(entries.get("attachments/report.txt")).toEqual(Buffer.from([0x00, 0x01, 0xff])); + expect(entries.get("attachments/report-2.txt")?.toString("utf8")).toBe("second"); + expect(entries.get("documents/plan.md")?.toString("utf8")).toBe("# Plan\n"); + expect(entries.has("documents/continuation-summary.md")).toBe(false); + expect(mockDocumentService.listIssueDocuments).toHaveBeenCalledWith("11111111-1111-4111-8111-111111111111"); + expect(storage.getObject).toHaveBeenCalledTimes(3); + }); + + it("sanitizes traversal and control characters in archive entry names", async () => { + const storage = createStorageService(Buffer.from("safe")); + mockIssueService.listAttachments.mockResolvedValue([{ + ...makeAttachment("text/plain", "../folder\\bad\u0000.txt"), + id: "attachment-safe", + }]); + + const app = await createApp(storage); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive") + .buffer(true) + .parse(parseBinaryResponse); + + expect(res.status).toBe(200); + const [entryName] = readZipEntries(Buffer.from(res.body)).keys(); + expect(entryName).toMatch(/^attachments\//); + expect(entryName).not.toContain(".."); + expect(entryName).not.toContain("\\"); + expect(entryName).not.toContain("\u0000"); + }); + + it("returns 422 when an issue has no exportable archive entries", async () => { + const app = await createApp(createStorageService()); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive"); + + expect(res.status).toBe(422); + expect(res.body.error).toBe("Issue has no exportable attachments or documents"); + }); + + it("returns 404 for a missing issue archive", async () => { + mockIssueService.getById.mockResolvedValue(null); + const app = await createApp(createStorageService()); + const res = await request(app).get("/api/issues/missing/attachments/archive"); + + expect(res.status).toBe(404); + }); + + it("returns the normal server error response when storage fails before streaming", async () => { + const storage = createStorageService(); + mockIssueService.listAttachments.mockResolvedValue([makeAttachment("text/plain", "report.txt")]); + vi.mocked(storage.getObject).mockRejectedValue(new Error("storage unavailable")); + const app = await createApp(storage); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive"); + + expect(res.status).toBe(500); + expect(res.headers["content-disposition"]).toBeUndefined(); + expect(res.body.error).toBe("Internal server error"); + }); + + it("rejects cross-company issue archive reads before loading entries", async () => { + const storage = createStorageService(); + const app = await createApp(storage, { companyIds: ["company-2"], source: "session" }); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive"); + + expect(res.status).toBe(403); + expect(mockIssueService.listAttachments).not.toHaveBeenCalled(); + expect(storage.getObject).not.toHaveBeenCalled(); + }); + + it("rejects issue archive reads outside the task boundary", async () => { + const storage = createStorageService(); + mockAccessService.decide.mockResolvedValue({ allowed: false, explanation: "Denied by test mock" }); + const app = await createApp(storage); + const res = await request(app) + .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive"); + + expect(res.status).toBe(403); + expect(mockIssueService.listAttachments).not.toHaveBeenCalled(); + expect(storage.getObject).not.toHaveBeenCalled(); + }); + it("accepts zip uploads for issue attachments", async () => { const storage = createStorageService(); mockIssueService.getById.mockResolvedValue({ diff --git a/server/src/issue-attachment-archive.ts b/server/src/issue-attachment-archive.ts new file mode 100644 index 00000000000..375cbb6c0f3 --- /dev/null +++ b/server/src/issue-attachment-archive.ts @@ -0,0 +1,88 @@ +import { extname } from "node:path"; + +type AttachmentArchiveInput = { + id: string; + contentType: string; + originalFilename: string | null; +}; + +type DocumentArchiveInput = { + key: string; +}; + +const CONTENT_TYPE_EXTENSIONS: Readonly> = { + "application/json": ".json", + "application/pdf": ".pdf", + "application/zip": ".zip", + "image/gif": ".gif", + "image/jpeg": ".jpg", + "image/jpg": ".jpg", + "image/png": ".png", + "image/svg+xml": ".svg", + "image/webp": ".webp", + "text/csv": ".csv", + "text/html": ".html", + "text/markdown": ".md", + "text/plain": ".txt", + "video/mp4": ".mp4", + "video/quicktime": ".mov", + "video/webm": ".webm", + "video/x-m4v": ".m4v", +}; + +export function sanitizeArchivePathSegment(value: string, fallback: string): string { + const sanitized = value + .normalize("NFKC") + .replace(/[\\/]/g, "-") + .replace(/[\u0000-\u001f\u007f]/g, "") + .replace(/\.{2,}/g, ".") + .replace(/^\.+/, "") + .trim(); + return sanitized || fallback; +} + +function inferredAttachmentExtension(contentType: string): string { + return CONTENT_TYPE_EXTENSIONS[contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""] ?? ""; +} + +function suffixDuplicateFilename(filename: string, suffix: number): string { + const extension = extname(filename); + const stem = extension ? filename.slice(0, -extension.length) : filename; + return `${stem}-${suffix}${extension}`; +} + +function allocateUniquePath(directory: string, filename: string, usedPaths: Set): string { + let candidate = `${directory}/${filename}`; + let suffix = 2; + while (usedPaths.has(candidate.toLowerCase())) { + candidate = `${directory}/${suffixDuplicateFilename(filename, suffix)}`; + suffix += 1; + } + usedPaths.add(candidate.toLowerCase()); + return candidate; +} + +export function buildAttachmentArchivePaths(attachments: AttachmentArchiveInput[]): Map { + const usedPaths = new Set(); + const paths = new Map(); + const sorted = [...attachments].sort((left, right) => left.id.localeCompare(right.id)); + + for (const attachment of sorted) { + const fallbackBase = sanitizeArchivePathSegment(`attachment-${attachment.id}`, "attachment"); + const fallback = `${fallbackBase}${inferredAttachmentExtension(attachment.contentType)}`; + const filename = sanitizeArchivePathSegment(attachment.originalFilename ?? "", fallback); + paths.set(attachment.id, allocateUniquePath("attachments", filename, usedPaths)); + } + + return paths; +} + +export function buildDocumentArchivePath(document: DocumentArchiveInput): string { + const key = sanitizeArchivePathSegment(document.key, "document"); + return `documents/${key.toLowerCase().endsWith(".md") ? key : `${key}.md`}`; +} + +export function buildIssueAttachmentArchiveFilename(identifier: string): string { + const safeIdentifier = sanitizeArchivePathSegment(identifier, "task").replace(/[";=]/g, "-"); + return `attachments-${safeIdentifier}.zip`; +} diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 4207e2d2c0b..3e6e36947e8 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,6 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; +import type { Readable } from "node:stream"; import { Router, type Request, type Response } from "express"; import multer from "multer"; +import ZipStream from "zip-stream"; import { z } from "zod"; import { and, asc, desc, eq, inArray, isNull, notInArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -248,6 +250,11 @@ import { observeCrossIssueInfluence, type CrossIssueInfluenceKind, } from "../services/cross-issue-influence-limit.js"; +import { + buildAttachmentArchivePaths, + buildDocumentArchivePath, + buildIssueAttachmentArchiveFilename, +} from "../issue-attachment-archive.js"; const MAX_ISSUE_COMMENT_LIMIT = 500; const updateIssueRouteSchema = updateIssueSchema.extend({ @@ -12620,6 +12627,99 @@ export function issueRoutes( res.json(attachments.map(withContentPath)); }); + router.get("/issues/:id/attachments/archive", async (req, res, next) => { + const issueId = req.params.id as string; + const issue = await svc.getById(issueId); + if (!issue) { + res.status(404).json({ error: "Issue not found" }); + return; + } + assertCompanyAccess(req, issue.companyId); + if (!(await assertIssueReadAllowed(req, res, issue))) return; + + const [attachments, issueDocuments] = await Promise.all([ + svc.listAttachments(issue.id), + documentsSvc.listIssueDocuments(issue.id), + ]); + if (attachments.length === 0 && issueDocuments.length === 0) { + res.status(422).json({ error: "Issue has no exportable attachments or documents" }); + return; + } + + const archive = new ZipStream({ zlib: { level: 9 } }); + const attachmentPaths = buildAttachmentArchivePaths(attachments); + const appendEntry = (source: Buffer | Readable, name: string, date: Date) => + new Promise((resolve, reject) => { + archive.entry(source, { name, date }, (err) => err ? reject(err) : resolve()); + }); + let handledError = false; + const handleArchiveError = (err: unknown, context: Record) => { + if (handledError) return; + handledError = true; + logger.error({ err, issueId: issue.id, ...context }, "failed to stream issue attachment archive"); + const error = err instanceof Error ? err : new Error(String(err)); + if (!res.headersSent) { + archive.unpipe(); + res.removeHeader("Content-Type"); + res.removeHeader("Content-Disposition"); + res.removeHeader("Cache-Control"); + res.removeHeader("X-Content-Type-Options"); + archive.destroy(); + next(error); + } else if (!res.destroyed) { + archive.destroy(error); + res.destroy(error); + } + }; + + archive.on("error", (err) => handleArchiveError(err, { stage: "archive" })); + res.on("close", () => { + if (!res.writableEnded && !archive.destroyed) archive.destroy(); + }); + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", `attachment; filename="${buildIssueAttachmentArchiveFilename(issue.identifier ?? issue.id)}"`); + res.setHeader("Cache-Control", "private, no-store, no-transform"); + res.setHeader("X-Content-Type-Options", "nosniff"); + archive.pipe(res); + + try { + for (const attachment of attachments) { + let object; + try { + object = await storage.getObject(attachment.companyId, attachment.objectKey); + } catch (err) { + logger.error( + { err, issueId: issue.id, attachmentId: attachment.id, objectKey: attachment.objectKey }, + "failed to read attachment while streaming issue archive", + ); + throw err; + } + object.stream.on("error", (err) => { + logger.error( + { err, issueId: issue.id, attachmentId: attachment.id, objectKey: attachment.objectKey }, + "attachment stream failed while streaming issue archive", + ); + }); + await appendEntry( + object.stream, + attachmentPaths.get(attachment.id) ?? `attachments/attachment-${attachment.id}`, + attachment.createdAt, + ); + } + + for (const document of issueDocuments) { + await appendEntry( + Buffer.from(document.body ?? "", "utf8"), + buildDocumentArchivePath(document), + document.updatedAt, + ); + } + archive.finalize(); + } catch (err) { + handleArchiveError(err, { stage: "entry" }); + } + }); + router.post("/companies/:companyId/issues/:issueId/attachments", async (req, res) => { const companyId = req.params.companyId as string; const issueId = req.params.issueId as string;