PaperClipAI/tests/runner-e2e/evidence.ts

import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, inspectPrivateFrameworkMetadata } from "./private-framework-report.js";
import { spawn } from "node:child_process";
import {
  mkdir,
  readFile,
  readdir,
  rm,
  writeFile,
  copyFile,
  mkdtemp,
  lstat,
} from "node:fs/promises";
import path from "node:path";
import os from "node:os";
import type { RunnerE2EResult } from "./types.js";
import { assertRunnerE2EResultsPath } from "./results-root.js";
import {
  assertSecretFree,
  findSecretLeak,
  findSecretLeakInJsonValues,
  redactText,
  sanitizeJson,
  normalizedSecrets,
} from "./redaction.js";

import { assertPrivateCellDirectories, cellTraceAsset, readPrivateCellTraceAsset, type PrivateCellEvidenceOwner } from "./manual-private-report-assets.js";

const TEXT_EXTENSIONS = new Set([
  ".json",
  ".xml",
  ".html",
  ".css",
  ".js",
  ".txt",
  ".log",
  ".md",
]);
const BINARY_EXTENSIONS = new Set([".png", ".webm"]);
const ACTIVE_CONTENT_EXTENSIONS = new Set([".svg"]);
const ALLOWED_DIRECTORIES = new Set([
  "snapshots",
  "playwright-output",
  "blob-report",
  "html-report",
]);
const ALLOWED_ROOT_FILES = new Set([
  "result.json",
  "final-state.png",
  "failure.png",
  "tool-review-pending.png",
  "decision-pending.png",

  "first-task-response.png",
  "chat-plan-draft.png",
  "chat-plan-revised.png",
  "server.log",
  "playwright.log",
  "junit.xml",
]);
const REQUIRED_PASS_FILES = [
  "final-state.png",
  "result.json",
  "junit.xml",
  path.join("html-report", "index.html"),
  path.join("snapshots", "fixtures.json"),
  path.join("snapshots", "api-state.json"),
] as const;

export function isStandardEvidenceFile(relative: string) {
  const extension = path.extname(relative).toLowerCase();
  return isStandardEvidencePath(relative) && (TEXT_EXTENSIONS.has(extension) || BINARY_EXTENSIONS.has(extension)
    || extension === ".zip");
}

// Diagnostic metadata only: never an admission, completeness or grading receipt.
// inventoryComplete describes the listed standard files, not a sealed original tree.
interface PrivateEvidenceBudget {
  bound: string; listedFiles: number; filesExamined: number; bytesExamined: number;
  inventoryComplete: boolean; scanFinished: boolean; requiredEvidenceComplete: boolean | null;
  byKind: Record<string, { files: number; bytes: number }>;
}
const evidenceBudgetKinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"];
function evidenceBudgetKind(extension: string): string {
  return extension === ".json" ? "json" : extension === ".log" ? "log" : [".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(extension) ? "image"
    : [".webm", ".mp4"].includes(extension) ? "video" : extension === ".zip" ? "zip" : extension === ".html" ? "html"
    : [".txt", ".md", ".xml"].includes(extension) ? "text" : "asset";
}
function budgetFailure(message: string, budget: PrivateEvidenceBudget, bound: string) {
  budget.bound = bound;
  return Object.assign(new Error(message), { evidenceBudget: budget });
}
function scannerBudgetBound(reason: string): string {
  return reason === "zip inspection exceeded its expanded-byte bound" ? "zip-expanded-bytes"
    : reason === "zip inspection exceeded its time bound" ? "zip-time"
    : reason === "embedded report exceeded its byte bound" ? "html-encoded-bytes" : "none";
}
export interface EvidencePackageResult {
  files: string[];
  leaks: Array<{ file: string; reason: string }>;
  budget?: PrivateEvidenceBudget;
  missing: string[];
}

async function walk(root: string, relative = "", privateStandardOnly = false): Promise<string[]> {
  const directory = path.join(root, relative);
  const entries = await readdir(directory, { withFileTypes: true }).catch(
    () => [],
  );
  const files: string[] = [];
  for (const entry of entries) {
    const next = path.join(relative, entry.name);
    // Installed preparation may leave a large, deliberately excluded staging
    // tree. Never enumerate it for private standard-evidence packaging.
    if (privateStandardOnly && !relative && !isStandardEvidencePath(entry.isDirectory() ? path.join(next, "placeholder") : next)) continue;
    if (entry.isDirectory()) files.push(...(await walk(root, next, privateStandardOnly)));
    else if (entry.isFile()) files.push(next);
  }
  return files;
}

export function isStandardEvidencePath(relative: string) {
  const segments = relative.split(path.sep);
  if (segments.length === 1)
    return (
      ALLOWED_ROOT_FILES.has(relative) ||
      /^(?:plan|question)-[a-z0-9-]+\.png$/.test(relative)
    );
  return ALLOWED_DIRECTORIES.has(segments[0]);
}

export async function inspectZip(source: string, secrets: readonly string[]) {
  // Failure traces can exceed Node's child-process output buffer. Stream the
  // expanded archive through the exact-value scanner with enough overlap to
  // detect a credential split across stdout chunks, without retaining the
  // archive in memory or weakening fail-closed evidence publication.
  const overlap = Math.max(
    256,
    ...secrets.map((secret) => Buffer.byteLength(secret, "utf8") + 16),
  );
  return new Promise<string | null>((resolve) => {
    const unzip = spawn("unzip", ["-p", source], {
      stdio: ["ignore", "pipe", "pipe"],
    });
    let carry = Buffer.alloc(0);
    let leak: string | null = null;
    let spawnError: Error | null = null;
    let expanded = 0;
    const timer = setTimeout(() => { leak = "zip inspection exceeded its time bound"; unzip.kill("SIGKILL"); }, 30_000);

    unzip.stdout.on("data", (chunk: Buffer) => {
      if (leak) return;
      expanded += chunk.length;
      if (expanded > 256 * 1024 * 1024) { leak = "zip inspection exceeded its expanded-byte bound"; unzip.kill("SIGKILL"); return; }
      const data = Buffer.concat([carry, chunk]);
      leak = findSecretLeak(data, secrets, { includeShapes: false });
      carry = data.subarray(Math.max(0, data.length - overlap));
      if (leak) unzip.kill("SIGKILL");
    });
    // Drain diagnostics so a noisy unzip cannot block. Error text is withheld
    // because archive paths and contents belong to private attempt evidence.
    unzip.stderr.resume();
    unzip.on("error", (error) => {
      spawnError = error;
    });
    unzip.on("close", (code) => {
      clearTimeout(timer);
      if (leak) return resolve(leak);
      if (spawnError) {
        return resolve(`zip could not be inspected: ${spawnError.message}`);
      }
      return resolve(
        code === 0
          ? null
          : `zip could not be inspected: unzip exited with code ${String(code)}`,
      );
    });
  });
}

/** Playwright HTML contains a compressed report, which a plaintext HTML scan
 * cannot inspect. It receives the same independent ZIP check as blob reports. */
export async function inspectEmbeddedReport(raw: string, secrets: readonly string[]) {
  const payloads = [...raw.matchAll(/data:application\/zip;base64,([A-Za-z0-9+/=]+)/g)];
  if (!payloads.length) return /playwrightReportBase64/.test(raw) ? "embedded report encoding could not be inspected" : null;
  const temporary = await mkdtemp(path.join(os.tmpdir(), "runner-report-scan-"));
  try {
    for (const [index, payload] of payloads.entries()) {
      if (payload[1]!.length > 128 * 1024 * 1024) return "embedded report exceeded its byte bound";
      const filename = path.join(temporary, `${index}.zip`);
      await writeFile(filename, Buffer.from(payload[1]!, "base64"), { mode: 0o600 });
      const issue = await inspectZip(filename, secrets); if (issue) return issue;
    }
    return null;
  } finally { await rm(temporary, { recursive: true, force: true }); }
}

export async function packageEvidence(input: {
  privateDir: string;
  uploadDir: string;
  secrets: readonly string[];
  expectPassScreenshot: boolean;
  privateCell?: { owner: PrivateCellEvidenceOwner; layout: "original" | "final-wrapper" };
}): Promise<EvidencePackageResult> {
  if (input.privateCell && !["original", "final-wrapper"].includes(input.privateCell.layout)) throw new Error("Invalid private cell packaging layout");
  if (input.privateCell) await assertPrivateCellDirectories(input.privateCell.owner, input.privateDir, input.uploadDir);
  await rm(input.uploadDir, { recursive: true, force: true });
  await mkdir(input.uploadDir, { recursive: true });
  const files: string[] = [];
  const leaks: EvidencePackageResult["leaks"] = [];
  const available = await walk(input.privateDir, "", Boolean(input.privateCell));
  const standard = available.filter(isStandardEvidencePath);

  const budget: PrivateEvidenceBudget | undefined = input.privateCell ? { bound: "none", listedFiles: Math.min(standard.length, 4097), filesExamined: 0, bytesExamined: 0,
    inventoryComplete: false, scanFinished: false, requiredEvidenceComplete: null,
    byKind: Object.fromEntries(evidenceBudgetKinds.map(kind => [kind, { files: 0, bytes: 0 }])) } : undefined;
  if (input.privateCell && standard.length > 4096) throw budgetFailure("Private cell evidence exceeds entry allowance", budget!, "entries");
  const privateSecrets = input.privateCell ? normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]) : [];
  const compact = input.privateCell?.layout === "original" && standard.includes(PRIVATE_FRAMEWORK_REPORT)
    ? await validatePrivateFrameworkReport(input.privateDir) : undefined;
  const registeredAttachments = new Set(compact?.attachments.map(row => row.path) ?? []);
  let privateBytes = 0;
  for (const relative of standard) {
    const source = path.join(input.privateDir, relative);
    const extension = path.extname(relative).toLowerCase();
    const destination = path.join(input.uploadDir, relative);
    if (input.privateCell) {
      assertSecretFree(relative, privateSecrets, "private cell evidence path");
      const metadata = await lstat(source);
      if (Number.isSafeInteger(metadata.size) && metadata.size >= 0 && Number.isSafeInteger(budget!.bytesExamined + metadata.size)) {
        budget!.filesExamined++; budget!.bytesExamined += metadata.size;
        const counter = budget!.byKind[evidenceBudgetKind(extension)]!; counter.files++; counter.bytes += metadata.size;
      }
      if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 256 * 1024 * 1024 - privateBytes)
        throw budgetFailure("Private cell evidence exceeds byte allowance", budget!, "physical-bytes");
      privateBytes += metadata.size;
      if (!Number.isSafeInteger(privateBytes)) throw new Error("Invalid private cell evidence byte accounting");
      const asset = cellTraceAsset(relative, input.privateCell.owner, input.privateCell.layout);
      if (asset.reserved) {
        const bytes = await readPrivateCellTraceAsset({ ...input.privateCell, relative, file: source, remaining: metadata.size, secrets: input.secrets });
        await mkdir(path.dirname(destination), { recursive: true, mode: 0o700 });
        await writeFile(destination, bytes!, { mode: 0o600 }); files.push(relative); continue;
      }
      if (!isStandardEvidenceFile(relative)) throw new Error("Unsupported private cell evidence carrier");
    }
    await mkdir(path.dirname(destination), { recursive: true });
    if (ACTIVE_CONTENT_EXTENSIONS.has(extension)) {
      // SVG can execute script when opened directly from an artifact. Keep the
      // source in the disposable private attempt directory, but never admit it
      // to the sanitized CI artifact.
      continue;
    } else if (TEXT_EXTENSIONS.has(extension)) {
      const raw = await readFile(source, "utf8");
      if (input.privateCell && inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets)) {
        assertSecretFree(raw, input.secrets, "private framework metadata");
        await copyFile(source, destination); files.push(relative); continue;
      }
      if (extension === ".html") {
        const issue = await inspectEmbeddedReport(raw, input.secrets);
        if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; }
      }
      const parsed = extension === ".json" ? JSON.parse(raw) : null;
      const leak =
        extension === ".json"
          ? findSecretLeakInJsonValues(parsed, input.secrets)
          : findSecretLeak(raw, input.secrets);
      if (leak) leaks.push({ file: relative, reason: leak });
      if (registeredAttachments.has(relative.split(path.sep).join("/"))) {
        if (extension === ".json" ? JSON.stringify(sanitizeJson(parsed, input.secrets)) !== JSON.stringify(parsed) : redactText(raw, input.secrets) !== raw) throw new Error("Private framework attachment requires redaction");
        if (!leak) { await copyFile(source, destination); files.push(relative); }
        continue;
      }
      // Redacting an already serialized JSON string can change escape
      // boundaries around shell commands. Sanitize parsed values instead so
      // uploaded snapshots stay valid JSON.
      const safe =
        extension === ".json"
          ? `${JSON.stringify(sanitizeJson(parsed, input.secrets), null, 2)}\n`
          : redactText(raw, input.secrets);
      if (extension === ".json") {
        const safeLeak = findSecretLeakInJsonValues(
          JSON.parse(safe),
          input.secrets,
        );
        if (safeLeak)
          throw new Error(`Secret leak in ${relative}: ${safeLeak}`);
      } else {
        assertSecretFree(safe, input.secrets, relative);
      }
      await writeFile(destination, safe, "utf8");
      files.push(relative);
    } else if (extension === ".zip") {
      const leak = await inspectZip(source, input.secrets);
      if (leak) {
        if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(leak);
        leaks.push({ file: relative, reason: leak });
        continue;
      }
      await copyFile(source, destination);
      files.push(relative);
    } else if (BINARY_EXTENSIONS.has(extension)) {
      // This raw-byte scan catches embedded plaintext credentials, but cannot
      // inspect rendered pixels. Provider/UI raster and video files remain in
      // the access-controlled CI artifact. The public publisher creates its
      // own synthetic summary image from fixed labels and numeric/status data.
      const raw = await readFile(source);
      const leak = findSecretLeak(raw, input.secrets);
      if (leak) {
        leaks.push({ file: relative, reason: leak });
        continue;
      }
      await copyFile(source, destination);
      files.push(relative);
    }
  }

  if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; }
  const missing = input.expectPassScreenshot
    ? [
        ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required) && !(compact && required === path.join("html-report", "index.html"))),
        ...(compact || files.some(
          (file) =>
            file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"),
        )
          ? []
          : [path.join("blob-report", "*.zip")]),
      ]
    : [];
  const manifest = {
    schema: "paperclip.runner-e2e.evidence/v1",
    files: [...files].sort(),
    leaks,
    missing,
  };
  const manifestText = `${JSON.stringify(sanitizeJson(manifest, input.secrets), null, 2)}\n`;
  const manifestLeak = findSecretLeakInJsonValues(
    JSON.parse(manifestText),
    input.secrets,
  );
  if (manifestLeak)
    throw new Error(`Secret leak in evidence-manifest.json: ${manifestLeak}`);
  await writeFile(
    path.join(input.uploadDir, "evidence-manifest.json"),
    manifestText,
    "utf8",
  );
  files.push("evidence-manifest.json");
  if (budget && input.expectPassScreenshot) budget.requiredEvidenceComplete = missing.length === 0 && leaks.length === 0;
  return { files, leaks, missing, ...(budget ? { budget } : {}) };
}

export async function publishRunnerE2EAttemptEvidence(input: {
  result: RunnerE2EResult;
  privateDir: string;
  uploadDir: string;
  resultsRoot: string;
  secrets: readonly string[];
}): Promise<RunnerE2EResult> {
  let result = input.result;
  const resultPath = path.join(input.privateDir, "result.json");
  const writeResult = () => writeFile(resultPath,
    `${JSON.stringify(sanitizeJson(result, input.secrets), null, 2)}\n`, "utf8");
  await assertRunnerE2EResultsPath(input.resultsRoot, input.uploadDir);
  await writeResult();
  const evidence = await packageEvidence({ ...input, expectPassScreenshot: result.status === "passed" });
  if (evidence.leaks.length > 0 || evidence.missing.length > 0) {
    result = {
      ...result,
      status: "failed",
      failureClass: evidence.leaks.length > 0 ? "secret_leak" : "permanent_infrastructure",
      error: evidence.leaks.length > 0
        ? `Secret leak rejected from evidence: ${evidence.leaks.map(leak => leak.file).join(", ")}`
        : `Required evidence missing: ${evidence.missing.join(", ")}`,
    };
    await writeResult();
    await assertRunnerE2EResultsPath(input.resultsRoot, input.uploadDir);
    await packageEvidence({ ...input, expectPassScreenshot: false });
  }
  return result;
}