PaperClipAI/tests/runner-e2e/evidence.ts

import { execFileSync } from "node:child_process";
import {
  mkdir,
  readFile,
  readdir,
  rm,
  writeFile,
  copyFile,
} from "node:fs/promises";
import path from "node:path";
import {
  assertSecretFree,
  findSecretLeak,
  findSecretLeakInJsonValues,
  redactText,
  sanitizeJson,
} from "./redaction.js";

const TEXT_EXTENSIONS = new Set([
  ".json",
  ".xml",
  ".html",
  ".css",
  ".js",
  ".txt",
  ".log",
  ".md",
]);
const BINARY_EXTENSIONS = new Set([".png", ".webm", ".svg"]);
const ALLOWED_DIRECTORIES = new Set([
  "snapshots",
  "playwright-output",
  "blob-report",
  "html-report",
]);
const ALLOWED_ROOT_FILES = new Set([
  "result.json",
  "final-state.png",
  "failure.png",
  "server.log",
  "playwright.log",
  "junit.xml",
]);
const REQUIRED_PASS_FILES = [
  "final-state.png",
  "result.json",
  "junit.xml",
  path.join("html-report", "index.html"),
  path.join("snapshots", "fixtures.json"),
  path.join("snapshots", "api-state.json"),
] as const;

export interface EvidencePackageResult {
  files: string[];
  leaks: Array<{ file: string; reason: string }>;
  missing: string[];
}

async function walk(root: string, relative = ""): Promise<string[]> {
  const directory = path.join(root, relative);
  const entries = await readdir(directory, { withFileTypes: true }).catch(
    () => [],
  );
  const files: string[] = [];
  for (const entry of entries) {
    const next = path.join(relative, entry.name);
    if (entry.isDirectory()) files.push(...(await walk(root, next)));
    else if (entry.isFile()) files.push(next);
  }
  return files;
}

function isAllowed(relative: string) {
  const segments = relative.split(path.sep);
  if (segments.length === 1)
    return (
      ALLOWED_ROOT_FILES.has(relative) ||
      /^plan-[a-z0-9-]+\.png$/.test(relative)
    );
  return ALLOWED_DIRECTORIES.has(segments[0]);
}

function inspectZip(source: string, secrets: readonly string[]) {
  try {
    const expanded = execFileSync("unzip", ["-p", source], {
      maxBuffer: 256 * 1024 * 1024,
    });
    // Traces include loaded application source and therefore deliberate fake
    // provider-key fixtures. ZIPs cannot be safely rewritten, so admit them
    // only after exact-value scanning against the credentials loaded for this
    // campaign; textual snapshots still receive shape scanning and redaction.
    return findSecretLeak(expanded, secrets, { includeShapes: false });
  } catch (error) {
    return `zip could not be inspected: ${error instanceof Error ? error.message : String(error)}`;
  }
}

export async function packageEvidence(input: {
  privateDir: string;
  uploadDir: string;
  secrets: readonly string[];
  expectPassScreenshot: boolean;
}): Promise<EvidencePackageResult> {
  await rm(input.uploadDir, { recursive: true, force: true });
  await mkdir(input.uploadDir, { recursive: true });
  const files: string[] = [];
  const leaks: EvidencePackageResult["leaks"] = [];
  const available = await walk(input.privateDir);

  for (const relative of available.filter(isAllowed)) {
    const source = path.join(input.privateDir, relative);
    const extension = path.extname(relative).toLowerCase();
    const destination = path.join(input.uploadDir, relative);
    await mkdir(path.dirname(destination), { recursive: true });
    if (TEXT_EXTENSIONS.has(extension)) {
      const raw = await readFile(source, "utf8");
      const parsed = extension === ".json" ? JSON.parse(raw) : null;
      const leak =
        extension === ".json"
          ? findSecretLeakInJsonValues(parsed, input.secrets)
          : findSecretLeak(raw, input.secrets);
      if (leak) leaks.push({ file: relative, reason: leak });
      // Redacting an already serialized JSON string can change escape
      // boundaries around shell commands. Sanitize parsed values instead so
      // uploaded snapshots stay valid JSON.
      const safe =
        extension === ".json"
          ? `${JSON.stringify(sanitizeJson(parsed, input.secrets), null, 2)}\n`
          : redactText(raw, input.secrets);
      if (extension === ".json") {
        const safeLeak = findSecretLeakInJsonValues(
          JSON.parse(safe),
          input.secrets,
        );
        if (safeLeak)
          throw new Error(`Secret leak in ${relative}: ${safeLeak}`);
      } else {
        assertSecretFree(safe, input.secrets, relative);
      }
      await writeFile(destination, safe, "utf8");
      files.push(relative);
    } else if (extension === ".zip") {
      const leak = inspectZip(source, input.secrets);
      if (leak) {
        leaks.push({ file: relative, reason: leak });
        continue;
      }
      await copyFile(source, destination);
      files.push(relative);
    } else if (BINARY_EXTENSIONS.has(extension)) {
      const raw = await readFile(source);
      const leak = findSecretLeak(raw, input.secrets);
      if (leak) {
        leaks.push({ file: relative, reason: leak });
        continue;
      }
      await copyFile(source, destination);
      files.push(relative);
    }
  }

  const missing = input.expectPassScreenshot
    ? [
        ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)),
        ...(files.some(
          (file) =>
            file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"),
        )
          ? []
          : [path.join("blob-report", "*.zip")]),
      ]
    : [];
  const manifest = {
    schema: "paperclip.runner-e2e.evidence/v1",
    files: [...files].sort(),
    leaks,
    missing,
  };
  const manifestText = `${JSON.stringify(sanitizeJson(manifest, input.secrets), null, 2)}\n`;
  const manifestLeak = findSecretLeakInJsonValues(
    JSON.parse(manifestText),
    input.secrets,
  );
  if (manifestLeak)
    throw new Error(`Secret leak in evidence-manifest.json: ${manifestLeak}`);
  await writeFile(
    path.join(input.uploadDir, "evidence-manifest.json"),
    manifestText,
    "utf8",
  );
  files.push("evidence-manifest.json");
  return { files, leaks, missing };
}