feat(server): stream issue attachment archives

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/package.json

@@ -79,6 +79,7 @@
     "sharp": "^0.35.3",
     "ssh2": "^1.17.0",
     "ws": "^8.21.1",
+    "zip-stream": "^7.0.5",
     "zod": "^3.24.2"
   },
   "devDependencies": {
@@ -90,6 +91,7 @@
     "@types/sharp": "^0.32.0",
     "@types/supertest": "^6.0.2",
     "@types/ws": "^8.18.1",
+    "@types/zip-stream": "^7.0.0",
     "cross-env": "^10.1.0",
     "supertest": "^7.0.0",
     "tsx": "^4.23.1",

server/src/__tests__/issue-attachment-routes.test.ts

@@ -1,5 +1,6 @@
 import { Readable } from "node:stream";
 import type { IncomingMessage } from "node:http";
+import { inflateRawSync } from "node:zlib";
 import express from "express";
 import request from "supertest";
 import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -10,6 +11,10 @@ const mockIssueService = vi.hoisted(() => ({
   getByIdentifier: vi.fn(),
   createAttachment: vi.fn(),
   getAttachmentById: vi.fn(),
+  listAttachments: vi.fn(),
+}));
+const mockDocumentService = vi.hoisted(() => ({
+  listIssueDocuments: vi.fn(),
 }));
 const mockCompanyService = vi.hoisted(() => ({
   getById: vi.fn(),
@@ -56,7 +61,7 @@ function registerRouteMocks() {
     companySkillService: () => ({}),
     companyService: () => mockCompanyService,
     documentAnnotationService: () => ({ remapOpenThreadsForDocument: async () => [] }),
-    documentService: () => ({}),
+    documentService: () => mockDocumentService,
     executionWorkspaceService: () => ({}),
     feedbackService: () => ({
       listIssueVotesForUser: vi.fn(async () => []),
@@ -204,6 +209,33 @@ function parseBinaryResponse(res: IncomingMessage, callback: (error: Error | nul
   res.on("error", callback);
 }
 
+function readZipEntries(buffer: Buffer): Map<string, Buffer> {
+  const endOfCentralDirectory = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
+  if (endOfCentralDirectory < 0) throw new Error("ZIP end-of-central-directory record not found");
+  const entryCount = buffer.readUInt16LE(endOfCentralDirectory + 10);
+  let centralOffset = buffer.readUInt32LE(endOfCentralDirectory + 16);
+  const entries = new Map<string, Buffer>();
+
+  for (let index = 0; index < entryCount; index += 1) {
+    if (buffer.readUInt32LE(centralOffset) !== 0x02014b50) throw new Error("Invalid ZIP central directory entry");
+    const compressionMethod = buffer.readUInt16LE(centralOffset + 10);
+    const compressedSize = buffer.readUInt32LE(centralOffset + 20);
+    const filenameLength = buffer.readUInt16LE(centralOffset + 28);
+    const extraLength = buffer.readUInt16LE(centralOffset + 30);
+    const commentLength = buffer.readUInt16LE(centralOffset + 32);
+    const localOffset = buffer.readUInt32LE(centralOffset + 42);
+    const filename = buffer.subarray(centralOffset + 46, centralOffset + 46 + filenameLength).toString("utf8");
+    const localFilenameLength = buffer.readUInt16LE(localOffset + 26);
+    const localExtraLength = buffer.readUInt16LE(localOffset + 28);
+    const dataOffset = localOffset + 30 + localFilenameLength + localExtraLength;
+    const compressed = buffer.subarray(dataOffset, dataOffset + compressedSize);
+    entries.set(filename, compressionMethod === 0 ? compressed : inflateRawSync(compressed));
+    centralOffset += 46 + filenameLength + extraLength + commentLength;
+  }
+
+  return entries;
+}
+
 describe("normalizeIssueAttachmentMaxBytes", () => {
   it("keeps the process-level attachment cap as the final cap", async () => {
     const previous = process.env.PAPERCLIP_ATTACHMENT_MAX_BYTES;
@@ -253,6 +285,8 @@ describe("issue attachment routes", () => {
       assigneeUserId: null,
       identifier: "PAP-1",
     });
+    mockIssueService.listAttachments.mockResolvedValue([]);
+    mockDocumentService.listIssueDocuments.mockResolvedValue([]);
     mockCompanyService.getById.mockResolvedValue({
       id: "company-1",
       attachmentMaxBytes: 1024 * 1024 * 1024,
@@ -262,6 +296,148 @@ describe("issue attachment routes", () => {
     mockWorkProductService.update.mockReset();
   });
 
+  it("streams attachments and visible documents in a private ZIP archive", async () => {
+    const storage = createStorageService();
+    const firstAttachment = {
+      ...makeAttachment("application/octet-stream", "report.txt"),
+      id: "attachment-a",
+      objectKey: "issues/issue-1/report-a.txt",
+      issueCommentId: "comment-1",
+    };
+    const duplicateAttachment = {
+      ...makeAttachment("application/octet-stream", "report.txt"),
+      id: "attachment-b",
+      objectKey: "issues/issue-1/report-b.txt",
+    };
+    const unnamedAttachment = {
+      ...makeAttachment("image/png", "ignored.png"),
+      id: "attachment-c",
+      objectKey: "issues/issue-1/unnamed",
+      originalFilename: null,
+    };
+    mockIssueService.listAttachments.mockResolvedValue([
+      duplicateAttachment,
+      unnamedAttachment,
+      firstAttachment,
+    ]);
+    mockDocumentService.listIssueDocuments.mockImplementation(async (_issueId, options) => options?.includeSystem
+      ? [
+          { key: "plan", body: "# Plan\n", updatedAt: new Date("2026-01-02T00:00:00.000Z") },
+          { key: "continuation-summary", body: "internal", updatedAt: new Date("2026-01-02T00:00:00.000Z") },
+        ]
+      : [{ key: "plan", body: "# Plan\n", updatedAt: new Date("2026-01-02T00:00:00.000Z") }]);
+    vi.mocked(storage.getObject).mockImplementation(async (_companyId, objectKey) => ({
+      stream: Readable.from({
+        "issues/issue-1/report-a.txt": Buffer.from([0x00, 0x01, 0xff]),
+        "issues/issue-1/report-b.txt": Buffer.from("second"),
+        "issues/issue-1/unnamed": Buffer.from([0x89, 0x50, 0x4e, 0x47]),
+      }[objectKey] ?? Buffer.alloc(0)),
+      contentLength: 0,
+    }));
+
+    const app = await createApp(storage);
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive")
+      .buffer(true)
+      .parse(parseBinaryResponse);
+
+    expect(res.status).toBe(200);
+    expect(res.headers["content-type"]).toContain("application/zip");
+    expect(res.headers["content-disposition"]).toBe('attachment; filename="attachments-PAP-1.zip"');
+    expect(res.headers["cache-control"]).toBe("private, no-store, no-transform");
+    expect(res.headers["x-content-type-options"]).toBe("nosniff");
+    expect(res.headers["content-length"]).toBeUndefined();
+    expect(res.headers["transfer-encoding"]).toBe("chunked");
+
+    const entries = readZipEntries(Buffer.from(res.body));
+    expect([...entries.keys()].sort()).toEqual([
+      "attachments/attachment-attachment-c.png",
+      "attachments/report-2.txt",
+      "attachments/report.txt",
+      "documents/plan.md",
+    ]);
+    expect(entries.get("attachments/report.txt")).toEqual(Buffer.from([0x00, 0x01, 0xff]));
+    expect(entries.get("attachments/report-2.txt")?.toString("utf8")).toBe("second");
+    expect(entries.get("documents/plan.md")?.toString("utf8")).toBe("# Plan\n");
+    expect(entries.has("documents/continuation-summary.md")).toBe(false);
+    expect(mockDocumentService.listIssueDocuments).toHaveBeenCalledWith("11111111-1111-4111-8111-111111111111");
+    expect(storage.getObject).toHaveBeenCalledTimes(3);
+  });
+
+  it("sanitizes traversal and control characters in archive entry names", async () => {
+    const storage = createStorageService(Buffer.from("safe"));
+    mockIssueService.listAttachments.mockResolvedValue([{
+      ...makeAttachment("text/plain", "../folder\\bad\u0000.txt"),
+      id: "attachment-safe",
+    }]);
+
+    const app = await createApp(storage);
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive")
+      .buffer(true)
+      .parse(parseBinaryResponse);
+
+    expect(res.status).toBe(200);
+    const [entryName] = readZipEntries(Buffer.from(res.body)).keys();
+    expect(entryName).toMatch(/^attachments\//);
+    expect(entryName).not.toContain("..");
+    expect(entryName).not.toContain("\\");
+    expect(entryName).not.toContain("\u0000");
+  });
+
+  it("returns 422 when an issue has no exportable archive entries", async () => {
+    const app = await createApp(createStorageService());
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive");
+
+    expect(res.status).toBe(422);
+    expect(res.body.error).toBe("Issue has no exportable attachments or documents");
+  });
+
+  it("returns 404 for a missing issue archive", async () => {
+    mockIssueService.getById.mockResolvedValue(null);
+    const app = await createApp(createStorageService());
+    const res = await request(app).get("/api/issues/missing/attachments/archive");
+
+    expect(res.status).toBe(404);
+  });
+
+  it("returns the normal server error response when storage fails before streaming", async () => {
+    const storage = createStorageService();
+    mockIssueService.listAttachments.mockResolvedValue([makeAttachment("text/plain", "report.txt")]);
+    vi.mocked(storage.getObject).mockRejectedValue(new Error("storage unavailable"));
+    const app = await createApp(storage);
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive");
+
+    expect(res.status).toBe(500);
+    expect(res.headers["content-disposition"]).toBeUndefined();
+    expect(res.body.error).toBe("Internal server error");
+  });
+
+  it("rejects cross-company issue archive reads before loading entries", async () => {
+    const storage = createStorageService();
+    const app = await createApp(storage, { companyIds: ["company-2"], source: "session" });
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive");
+
+    expect(res.status).toBe(403);
+    expect(mockIssueService.listAttachments).not.toHaveBeenCalled();
+    expect(storage.getObject).not.toHaveBeenCalled();
+  });
+
+  it("rejects issue archive reads outside the task boundary", async () => {
+    const storage = createStorageService();
+    mockAccessService.decide.mockResolvedValue({ allowed: false, explanation: "Denied by test mock" });
+    const app = await createApp(storage);
+    const res = await request(app)
+      .get("/api/issues/11111111-1111-4111-8111-111111111111/attachments/archive");
+
+    expect(res.status).toBe(403);
+    expect(mockIssueService.listAttachments).not.toHaveBeenCalled();
+    expect(storage.getObject).not.toHaveBeenCalled();
+  });
+
   it("accepts zip uploads for issue attachments", async () => {
     const storage = createStorageService();
     mockIssueService.getById.mockResolvedValue({

server/src/issue-attachment-archive.ts

@@ -0,0 +1,88 @@
+import { extname } from "node:path";
+
+type AttachmentArchiveInput = {
+  id: string;
+  contentType: string;
+  originalFilename: string | null;
+};
+
+type DocumentArchiveInput = {
+  key: string;
+};
+
+const CONTENT_TYPE_EXTENSIONS: Readonly<Record<string, string>> = {
+  "application/json": ".json",
+  "application/pdf": ".pdf",
+  "application/zip": ".zip",
+  "image/gif": ".gif",
+  "image/jpeg": ".jpg",
+  "image/jpg": ".jpg",
+  "image/png": ".png",
+  "image/svg+xml": ".svg",
+  "image/webp": ".webp",
+  "text/csv": ".csv",
+  "text/html": ".html",
+  "text/markdown": ".md",
+  "text/plain": ".txt",
+  "video/mp4": ".mp4",
+  "video/quicktime": ".mov",
+  "video/webm": ".webm",
+  "video/x-m4v": ".m4v",
+};
+
+export function sanitizeArchivePathSegment(value: string, fallback: string): string {
+  const sanitized = value
+    .normalize("NFKC")
+    .replace(/[\\/]/g, "-")
+    .replace(/[\u0000-\u001f\u007f]/g, "")
+    .replace(/\.{2,}/g, ".")
+    .replace(/^\.+/, "")
+    .trim();
+  return sanitized || fallback;
+}
+
+function inferredAttachmentExtension(contentType: string): string {
+  return CONTENT_TYPE_EXTENSIONS[contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""] ?? "";
+}
+
+function suffixDuplicateFilename(filename: string, suffix: number): string {
+  const extension = extname(filename);
+  const stem = extension ? filename.slice(0, -extension.length) : filename;
+  return `${stem}-${suffix}${extension}`;
+}
+
+function allocateUniquePath(directory: string, filename: string, usedPaths: Set<string>): string {
+  let candidate = `${directory}/${filename}`;
+  let suffix = 2;
+  while (usedPaths.has(candidate.toLowerCase())) {
+    candidate = `${directory}/${suffixDuplicateFilename(filename, suffix)}`;
+    suffix += 1;
+  }
+  usedPaths.add(candidate.toLowerCase());
+  return candidate;
+}
+
+export function buildAttachmentArchivePaths(attachments: AttachmentArchiveInput[]): Map<string, string> {
+  const usedPaths = new Set<string>();
+  const paths = new Map<string, string>();
+  const sorted = [...attachments].sort((left, right) => left.id.localeCompare(right.id));
+
+  for (const attachment of sorted) {
+    const fallbackBase = sanitizeArchivePathSegment(`attachment-${attachment.id}`, "attachment");
+    const fallback = `${fallbackBase}${inferredAttachmentExtension(attachment.contentType)}`;
+    const filename = sanitizeArchivePathSegment(attachment.originalFilename ?? "", fallback);
+    paths.set(attachment.id, allocateUniquePath("attachments", filename, usedPaths));
+  }
+
+  return paths;
+}
+
+export function buildDocumentArchivePath(document: DocumentArchiveInput): string {
+  const key = sanitizeArchivePathSegment(document.key, "document");
+  return `documents/${key.toLowerCase().endsWith(".md") ? key : `${key}.md`}`;
+}
+
+export function buildIssueAttachmentArchiveFilename(identifier: string): string {
+  const safeIdentifier = sanitizeArchivePathSegment(identifier, "task").replace(/[";=]/g, "-");
+  return `attachments-${safeIdentifier}.zip`;
+}

server/src/routes/issues.ts

@@ -1,6 +1,8 @@
 import { createHash, randomUUID } from "node:crypto";
+import type { Readable } from "node:stream";
 import { Router, type Request, type Response } from "express";
 import multer from "multer";
+import ZipStream from "zip-stream";
 import { z } from "zod";
 import { and, asc, desc, eq, inArray, isNull, notInArray } from "drizzle-orm";
 import type { Db } from "@paperclipai/db";
@@ -248,6 +250,11 @@ import {
   observeCrossIssueInfluence,
   type CrossIssueInfluenceKind,
 } from "../services/cross-issue-influence-limit.js";
+import {
+  buildAttachmentArchivePaths,
+  buildDocumentArchivePath,
+  buildIssueAttachmentArchiveFilename,
+} from "../issue-attachment-archive.js";
 
 const MAX_ISSUE_COMMENT_LIMIT = 500;
 const updateIssueRouteSchema = updateIssueSchema.extend({
@@ -12620,6 +12627,99 @@ export function issueRoutes(
     res.json(attachments.map(withContentPath));
   });
 
+  router.get("/issues/:id/attachments/archive", async (req, res, next) => {
+    const issueId = req.params.id as string;
+    const issue = await svc.getById(issueId);
+    if (!issue) {
+      res.status(404).json({ error: "Issue not found" });
+      return;
+    }
+    assertCompanyAccess(req, issue.companyId);
+    if (!(await assertIssueReadAllowed(req, res, issue))) return;
+
+    const [attachments, issueDocuments] = await Promise.all([
+      svc.listAttachments(issue.id),
+      documentsSvc.listIssueDocuments(issue.id),
+    ]);
+    if (attachments.length === 0 && issueDocuments.length === 0) {
+      res.status(422).json({ error: "Issue has no exportable attachments or documents" });
+      return;
+    }
+
+    const archive = new ZipStream({ zlib: { level: 9 } });
+    const attachmentPaths = buildAttachmentArchivePaths(attachments);
+    const appendEntry = (source: Buffer | Readable, name: string, date: Date) =>
+      new Promise<void>((resolve, reject) => {
+        archive.entry(source, { name, date }, (err) => err ? reject(err) : resolve());
+      });
+    let handledError = false;
+    const handleArchiveError = (err: unknown, context: Record<string, unknown>) => {
+      if (handledError) return;
+      handledError = true;
+      logger.error({ err, issueId: issue.id, ...context }, "failed to stream issue attachment archive");
+      const error = err instanceof Error ? err : new Error(String(err));
+      if (!res.headersSent) {
+        archive.unpipe();
+        res.removeHeader("Content-Type");
+        res.removeHeader("Content-Disposition");
+        res.removeHeader("Cache-Control");
+        res.removeHeader("X-Content-Type-Options");
+        archive.destroy();
+        next(error);
+      } else if (!res.destroyed) {
+        archive.destroy(error);
+        res.destroy(error);
+      }
+    };
+
+    archive.on("error", (err) => handleArchiveError(err, { stage: "archive" }));
+    res.on("close", () => {
+      if (!res.writableEnded && !archive.destroyed) archive.destroy();
+    });
+    res.setHeader("Content-Type", "application/zip");
+    res.setHeader("Content-Disposition", `attachment; filename="${buildIssueAttachmentArchiveFilename(issue.identifier ?? issue.id)}"`);
+    res.setHeader("Cache-Control", "private, no-store, no-transform");
+    res.setHeader("X-Content-Type-Options", "nosniff");
+    archive.pipe(res);
+
+    try {
+      for (const attachment of attachments) {
+        let object;
+        try {
+          object = await storage.getObject(attachment.companyId, attachment.objectKey);
+        } catch (err) {
+          logger.error(
+            { err, issueId: issue.id, attachmentId: attachment.id, objectKey: attachment.objectKey },
+            "failed to read attachment while streaming issue archive",
+          );
+          throw err;
+        }
+        object.stream.on("error", (err) => {
+          logger.error(
+            { err, issueId: issue.id, attachmentId: attachment.id, objectKey: attachment.objectKey },
+            "attachment stream failed while streaming issue archive",
+          );
+        });
+        await appendEntry(
+          object.stream,
+          attachmentPaths.get(attachment.id) ?? `attachments/attachment-${attachment.id}`,
+          attachment.createdAt,
+        );
+      }
+
+      for (const document of issueDocuments) {
+        await appendEntry(
+          Buffer.from(document.body ?? "", "utf8"),
+          buildDocumentArchivePath(document),
+          document.updatedAt,
+        );
+      }
+      archive.finalize();
+    } catch (err) {
+      handleArchiveError(err, { stage: "entry" });
+    }
+  });
+
   router.post("/companies/:companyId/issues/:issueId/attachments", async (req, res) => {
     const companyId = req.params.companyId as string;
     const issueId = req.params.issueId as string;