PaperClipAI/tests/runner-e2e/history-publish.ts

import { validateRetainedRunnerResult } from "./result-validation.js";
import { createHash } from "node:crypto";
import { execFile } from "node:child_process";
import path from "node:path";
import { promisify } from "node:util";
import {
  cp,
  copyFile,
  mkdtemp,
  lstat,
  mkdir,
  readFile,
  readdir,
  realpath,
  rm,
  stat,
  writeFile,
} from "node:fs/promises";
import os from "node:os";
import { writePublicCampaignSummaryImage } from "./public-summary-image.js";
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
import { renderRunnerHistoryIndex } from "./history-index.js";
import { validateHistoryDestination } from "./history-destination.js";
import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js";
import {
  campaignHistoryRecord,
  emptyRunnerHistory,
  mergeRunnerHistory,
} from "./history.js";
import type { RunnerE2ECampaign, RunnerE2EHistoryIndex } from "./types.js";

const execFileAsync = promisify(execFile);
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const MUTABLE_HISTORY_FILES = new Set([
  "history.json",
  "latest.json",
  "latest-green.json",
]);
const PUBLISH_ROOT_FILES = new Set([
  "dashboard.html",
  "index.html",
  "junit.xml",
  "normalized-results.json",
  "summary.md",
]);
const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]);
const MAX_PUBLIC_RASTER_BYTES = 12 * 1024 * 1024;
const EMPTY_PUBLIC_SCREENSHOTS: ReadonlySet<string> = new Set();
const PRIVATE_EVIDENCE_DIRECTORIES = new Set([
  "blob-report",
  "html-report",
  "playwright-output",
]);

function publicEvidencePath(
  relative: string,
  publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
) {
  const match = relative.match(
    /^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/,
  );
  if (!match) return false;
  const evidencePath = match[1]!;
  const segments = evidencePath.split("/");
  if (
    segments.some(
      (segment) =>
        !segment ||
        segment === "." ||
        segment === ".." ||
        PRIVATE_EVIDENCE_DIRECTORIES.has(segment),
    )
  ) {
    return false;
  }
  return (
    PUBLIC_EVIDENCE_EXTENSIONS.has(
      path.posix.extname(evidencePath).toLowerCase(),
    ) || publicScreenshots.has(relative)
  );
}

export function publicScreenshotPaths(campaign: RunnerE2ECampaign) {
  const screenshots = new Set<string>();
  for (const result of campaign.results) {
    const files =
      result.screenshots
        ?.filter(
          (screenshot) =>
            screenshot.publication === PUBLIC_RUNNER_SCREENSHOT_MARKER,
        )
        .map((screenshot) => screenshot.file) ?? [];
    if (files.length === 0) continue;
    if (
      !/^[A-Za-z0-9][A-Za-z0-9._-]{0,299}$/.test(result.executionId) ||
      !Number.isSafeInteger(result.attempt) ||
      result.attempt < 1
    ) {
      throw new Error(
        `Cannot publish screenshots for unsafe execution identity ${result.executionId}`,
      );
    }
    for (const file of files) {
      if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.png$/i.test(file)) {
        throw new Error(
          `Cannot publish unsafe screenshot path for ${result.executionId}: ${file}`,
        );
      }
      screenshots.add(
        `evidence/${result.executionId}/attempt-${result.attempt}/${file}`,
      );
    }
  }
  return screenshots;
}

export function isHistoricalBundlePathAllowed(
  relative: string,
  allowPublicSummary = false,
  publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
) {
  if (
    relative.includes("\\") ||
    relative.startsWith("/") ||
    relative.includes("..")
  ) {
    return false;
  }
  if (PUBLISH_ROOT_FILES.has(relative)) return true;
  if (allowPublicSummary && relative === "public-images/campaign-summary.png") {
    return true;
  }
  if (
    relative === "assets/favicon-32x32.png" ||
    relative === "assets/InterVariable.woff2"
  ) {
    return true;
  }
  return publicEvidencePath(relative, publicScreenshots);
}

function hasPublicPngMagic(content: Buffer) {
  return (
    content.length >= 24 &&
    content
      .subarray(0, 8)
      .equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) &&
    content.subarray(12, 16).equals(Buffer.from("IHDR", "ascii"))
  );
}

async function validatePublicRaster(absolute: string, relative: string) {
  const metadata = await stat(absolute);
  if (metadata.size === 0 || metadata.size > MAX_PUBLIC_RASTER_BYTES) {
    throw new Error(
      `Public screenshot ${relative} exceeds the per-file size boundary`,
    );
  }
  const content = await readFile(absolute);
  if (!hasPublicPngMagic(content)) {
    throw new Error(
      `Public screenshot ${relative} does not match its raster file type`,
    );
  }
}

async function pruneEvidenceDirectory(
  root: string,
  current: string,
  publicScreenshots: ReadonlySet<string>,
) {
  const entries = await readdir(current, { withFileTypes: true });
  for (const entry of entries) {
    const absolute = path.join(current, entry.name);
    if (entry.isDirectory()) {
      await pruneEvidenceDirectory(root, absolute, publicScreenshots);
      if ((await readdir(absolute)).length === 0) {
        await rm(absolute, { recursive: true });
      }
      continue;
    }
    const relative = path.relative(root, absolute).split(path.sep).join("/");
    if (!entry.isFile() || !publicEvidencePath(relative, publicScreenshots)) {
      await rm(absolute, { force: true });
    }
  }
}

export async function prunePrivateHistoryEvidence(
  root: string,
  publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
) {
  const evidenceRoot = path.join(root, "evidence");
  const metadata = await lstat(evidenceRoot).catch(() => null);
  if (!metadata) return;
  if (!metadata.isDirectory()) {
    throw new Error("Historical evidence root must be a directory");
  }
  await pruneEvidenceDirectory(root, evidenceRoot, publicScreenshots);
}

export async function stageTrustedHistoryAssets(
  root: string,
  trustedRoot = repositoryRoot,
) {
  const resolveTrustedAsset = async (segments: string[]) => {
    const trustedRootReal = await realpath(trustedRoot);
    let source = trustedRoot;
    for (const [index, segment] of segments.entries()) {
      source = path.join(source, segment);
      const metadata = await lstat(source);
      if (metadata.isSymbolicLink()) {
        throw new Error(
          `Refusing symbolic link in trusted publisher asset path ${segments.join("/")}`,
        );
      }
      const final = index === segments.length - 1;
      if (
        (final && !metadata.isFile()) ||
        (!final && !metadata.isDirectory())
      ) {
        throw new Error(
          `Trusted publisher asset path has an invalid file type: ${segments.join("/")}`,
        );
      }
    }
    const sourceReal = await realpath(source);
    const relative = path.relative(trustedRootReal, sourceReal);
    if (
      !relative ||
      relative === ".." ||
      relative.startsWith(`..${path.sep}`)
    ) {
      throw new Error(
        `Trusted publisher asset escapes its checkout: ${segments.join("/")}`,
      );
    }
    return sourceReal;
  };
  const [faviconSource, fontSource] = await Promise.all([
    resolveTrustedAsset(["ui", "public", "favicon-32x32.png"]),
    resolveTrustedAsset(["ui", "public", "fonts", "InterVariable.woff2"]),
  ]);
  const assets = path.join(root, "assets");
  await rm(assets, { recursive: true, force: true });
  await mkdir(assets, { recursive: true });
  await Promise.all([
    copyFile(faviconSource, path.join(assets, "favicon-32x32.png")),
    copyFile(fontSource, path.join(assets, "InterVariable.woff2")),
  ]);
}

interface BundleManifest {
  schema: "paperclip.runner-e2e.bundle/v1";
  campaignId: string;
  bundleDigest: string;
  files: Array<{ path: string; sha256: string; bytes: number }>;
}

function json(value: unknown) {
  return `${JSON.stringify(value, null, 2)}\n`;
}

export { validateHistoryDestination };

async function relativeFiles(
  root: string,
  current = root,
  allowPublicSummary = false,
  publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
): Promise<string[]> {
  const entries = await readdir(current, { withFileTypes: true });
  const files: string[] = [];
  for (const entry of entries) {
    const absolute = path.join(current, entry.name);
    if (entry.isSymbolicLink()) {
      throw new Error(`Refusing to publish symbolic link ${entry.name}`);
    }
    if (entry.isDirectory()) {
      files.push(
        ...(await relativeFiles(
          root,
          absolute,
          allowPublicSummary,
          publicScreenshots,
        )),
      );
    } else if (entry.isFile()) {
      const relative = path.relative(root, absolute).split(path.sep).join("/");
      if (MUTABLE_HISTORY_FILES.has(relative)) continue;
      if (
        !isHistoricalBundlePathAllowed(
          relative,
          allowPublicSummary,
          publicScreenshots,
        )
      ) {
        throw new Error(
          `Refusing non-allowlisted historical bundle path ${relative}`,
        );
      }
      files.push(relative);
    }
  }
  return files;
}

export async function createBundleManifest(
  root: string,
  campaignId: string,
  allowPublicSummary = false,
  publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
): Promise<BundleManifest> {
  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) {
    throw new Error("Campaign ID is unsafe for immutable object storage");
  }
  const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots);
  const retained = new Set(retainedPaths);
  const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative));
  if (missingScreenshots.length > 0) {
    throw new Error(`Declared public screenshots are missing from the historical bundle: ${missingScreenshots.sort().join(", ")}`);
  }
  const files = await Promise.all(
    retainedPaths.sort().map(async (relative) => {
        const absolute = path.join(root, ...relative.split("/"));
        if (
          relative === "public-images/campaign-summary.png" ||
          publicScreenshots.has(relative)
        ) {
          await validatePublicRaster(absolute, relative);
        }
        const [content, metadata] = await Promise.all([
          readFile(absolute),
          stat(absolute),
        ]);
        return {
          path: relative,
          sha256: createHash("sha256").update(content).digest("hex"),
          bytes: metadata.size,
        };
      }),
  );
  const bundleDigest = createHash("sha256")
    .update(JSON.stringify(files))
    .digest("hex");
  return {
    schema: "paperclip.runner-e2e.bundle/v1",
    campaignId,
    bundleDigest,
    files,
  };
}

export function buildHistoryPointers(history: RunnerE2EHistoryIndex) {
  const byCampaign = new Map(
    history.campaigns.map((campaign) => [campaign.campaignId, campaign]),
  );
  const pointer = (campaignId: string | null | undefined) => {
    const campaign = campaignId ? byCampaign.get(campaignId) : undefined;
    return campaign
      ? {
          campaignId: campaign.campaignId,
          generatedAt: campaign.generatedAt,
          publicUrl: campaign.publicUrl,
          sha: campaign.source.sha,
        }
      : null;
  };
  return {
    latest: {
      schema: "paperclip.runner-e2e.pointer/v1",
      updatedAt: history.updatedAt,
      overall: pointer(history.latestCampaignId),
      suites: Object.fromEntries(
        Object.entries(history.latestBySuite).map(([suiteId, campaignId]) => [
          suiteId,
          pointer(campaignId),
        ]),
      ),
    },
    latestGreen: {
      schema: "paperclip.runner-e2e.pointer/v1",
      updatedAt: history.updatedAt,
      overall: pointer(history.latestGreenCampaignId),
      suites: Object.fromEntries(
        Object.entries(history.latestGreenBySuite).map(
          ([suiteId, campaignId]) => [suiteId, pointer(campaignId)],
        ),
      ),
    },
  };
}

function awsObject(bucket: string, key: string) {
  return `s3://${bucket}/${key}`;
}

async function objectExists(bucket: string, key: string) {
  try {
    await execFileAsync("aws", [
      "s3api",
      "head-object",
      "--bucket",
      bucket,
      "--key",
      key,
    ]);
    return true;
  } catch (error) {
    const detail = String(
      (error as { stderr?: string }).stderr ??
        (error instanceof Error ? error.message : error),
    );
    if (/\b(?:404|Not Found|NoSuchKey)\b/i.test(detail)) return false;
    throw new Error(
      `Unable to inspect historical object: ${detail.slice(0, 400)}`,
    );
  }
}

async function downloadJson<T>(
  bucket: string,
  key: string,
  destination: string,
) {
  if (!(await objectExists(bucket, key))) return null;
  await execFileAsync("aws", [
    "s3",
    "cp",
    awsObject(bucket, key),
    destination,
    "--only-show-errors",
  ]);
  return JSON.parse(await readFile(destination, "utf8")) as T;
}

async function uploadJson(
  bucket: string,
  key: string,
  file: string,
  cacheControl: string,
) {
  await uploadFile(bucket, key, file, "application/json", cacheControl);
}

async function uploadFile(
  bucket: string,
  key: string,
  file: string,
  contentType: string,
  cacheControl: string,
) {
  await execFileAsync("aws", [
    "s3",
    "cp",
    file,
    awsObject(bucket, key),
    "--only-show-errors",
    "--content-type",
    contentType,
    "--cache-control",
    cacheControl,
  ]);
}

async function uploadImmutableBundle(
  bucket: string,
  key: string,
  directory: string,
) {
  const common = [
    "--recursive",
    "--only-show-errors",
    "--cache-control",
    "public,max-age=31536000,immutable",
  ];
  await execFileAsync("aws", [
    "s3",
    "cp",
    directory,
    awsObject(bucket, key),
    ...common,
    "--exclude",
    "history.json",
    "--exclude",
    "latest.json",
    "--exclude",
    "latest-green.json",
    "--exclude",
    "*.png",
  ]);
  await execFileAsync("aws", [
    "s3",
    "cp",
    directory,
    awsObject(bucket, key),
    ...common,
    "--exclude",
    "*",
    "--include",
    "*.png",
    "--content-type",
    "image/png",
  ]);
}

async function main() {
  const reportRoot = path.resolve(
    process.env.PAPERCLIP_RUNNER_E2E_REPORT_DIR ??
      "runner-e2e-merged-report/normalized",
  );
  const bucket = process.env.RUNNER_E2E_HISTORY_S3_BUCKET ?? "";
  const destination = validateHistoryDestination({
    bucket,
    prefix: process.env.RUNNER_E2E_HISTORY_PREFIX ?? "runner-e2e",
    publicBaseUrl: process.env.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL ?? "",
  });
  const campaign = JSON.parse(
    await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"),
  ) as RunnerE2ECampaign;
  for (const result of campaign.results ?? []) validateRetainedRunnerResult(result);
  if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") {
    throw new Error("Historical publishing requires a v2 normalized campaign");
  }

  const temporary = await mkdtemp(
    path.join(os.tmpdir(), "runner-e2e-history-"),
  );
  const historyKey = `${destination.prefix}/history.json`;
  const current =
    (await downloadJson<RunnerE2EHistoryIndex>(
      bucket,
      historyKey,
      path.join(temporary, "current-history.json"),
    )) ?? emptyRunnerHistory();
  const history = mergeRunnerHistory(
    current,
    campaignHistoryRecord(
      campaign,
      `${destination.publicBaseUrl}/${destination.prefix}`,
    ),
  );
  const publicScreenshots = publicScreenshotPaths(campaign);

  // Campaign bundles are immutable and must not capture a mutable history
  // file left in a reused local directory. The root landing page below is the
  // only dashboard that embeds navigation across campaigns. S3 gets its own
  // staged copy. Only screenshots declared by normalized results (plus the
  // fixed failure screenshot for failed executions) cross the public boundary;
  // all other target-produced files remain subject to the structured allowlist.
  const s3ReportRoot = path.join(temporary, "s3-campaign");
  await cp(reportRoot, s3ReportRoot, { recursive: true, errorOnExist: true });
  await prunePrivateHistoryEvidence(s3ReportRoot, publicScreenshots);
  await stageTrustedHistoryAssets(s3ReportRoot);
  await rm(path.join(s3ReportRoot, "public-images"), {
    recursive: true,
    force: true,
  });
  const summaryImage = path.join(
    s3ReportRoot,
    "public-images",
    "campaign-summary.png",
  );
  await writePublicCampaignSummaryImage(campaign, summaryImage);
  await regenerateRunnerDashboard({
    bundle: s3ReportRoot,
    historyFile: null,
    publicSummaryImageHref: "public-images/campaign-summary.png",
  });
  const manifest = await createBundleManifest(
    s3ReportRoot,
    campaign.campaignId,
    true,
    publicScreenshots,
  );
  const campaignPrefix = `${destination.prefix}/campaigns/${campaign.campaignId}`;
  const manifestKey = `${campaignPrefix}/bundle-manifest.json`;
  const existingManifest = await downloadJson<BundleManifest>(
    bucket,
    manifestKey,
    path.join(temporary, "existing-manifest.json"),
  );
  if (
    existingManifest &&
    existingManifest.bundleDigest !== manifest.bundleDigest
  ) {
    throw new Error(
      `Immutable campaign ${campaign.campaignId} already exists with a different digest`,
    );
  }
  if (!existingManifest) {
    await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot);
    const manifestFile = path.join(temporary, "bundle-manifest.json");
    await writeFile(manifestFile, json(manifest), "utf8");
    await uploadJson(
      bucket,
      manifestKey,
      manifestFile,
      "public,max-age=31536000,immutable",
    );
  }

  // Pages uses the same declared-screenshot boundary in a sibling stage. The
  // downloaded access-controlled report stays intact for this whole job.
  const pagesRoot = path.join(path.dirname(reportRoot), "pages");
  await rm(pagesRoot, { recursive: true, force: true });
  await cp(reportRoot, pagesRoot, { recursive: true, errorOnExist: true });
  await prunePrivateHistoryEvidence(pagesRoot, publicScreenshots);
  await stageTrustedHistoryAssets(pagesRoot);
  await rm(path.join(pagesRoot, "public-images"), {
    recursive: true,
    force: true,
  });
  const pointers = buildHistoryPointers(history);
  const historyFile = path.join(pagesRoot, "history.json");
  const latestFile = path.join(pagesRoot, "latest.json");
  const latestGreenFile = path.join(pagesRoot, "latest-green.json");
  await Promise.all([
    writeFile(historyFile, json(history), "utf8"),
    writeFile(latestFile, json(pointers.latest), "utf8"),
    writeFile(latestGreenFile, json(pointers.latestGreen), "utf8"),
  ]);
  await regenerateRunnerDashboard({ bundle: pagesRoot, historyFile });
  await createBundleManifest(
    pagesRoot,
    campaign.campaignId,
    false,
    publicScreenshots,
  );
  const landingDirectory = path.join(temporary, "landing");
  await regenerateRunnerDashboard({
    bundle: s3ReportRoot,
    historyFile,
    outputDirectory: landingDirectory,
    evidenceHrefPrefix: `campaigns/${campaign.campaignId}`,
    publicSummaryImageHref: `campaigns/${campaign.campaignId}/public-images/campaign-summary.png`,
  });
  await writeFile(
    path.join(landingDirectory, "index.html"),
    renderRunnerHistoryIndex(history, {
      latestSummaryImageHref:
        history.latestCampaignId === campaign.campaignId
          ? `campaigns/${campaign.campaignId}/public-images/campaign-summary.png`
          : undefined,
    }),
    "utf8",
  );
  await Promise.all([
    uploadJson(bucket, historyKey, historyFile, "no-cache"),
    uploadJson(
      bucket,
      `${destination.prefix}/latest.json`,
      latestFile,
      "no-cache",
    ),
    uploadJson(
      bucket,
      `${destination.prefix}/latest-green.json`,
      latestGreenFile,
      "no-cache",
    ),
    uploadFile(
      bucket,
      `${destination.prefix}/index.html`,
      path.join(landingDirectory, "index.html"),
      "text/html; charset=utf-8",
      "no-cache",
    ),
    uploadFile(
      bucket,
      `${destination.prefix}/dashboard.html`,
      path.join(landingDirectory, "dashboard.html"),
      "text/html; charset=utf-8",
      "no-cache",
    ),
    uploadFile(
      bucket,
      `${destination.prefix}/normalized-results.json`,
      path.join(landingDirectory, "normalized-results.json"),
      "application/json",
      "no-cache",
    ),
    uploadFile(
      bucket,
      `${destination.prefix}/assets/favicon-32x32.png`,
      path.join(s3ReportRoot, "assets", "favicon-32x32.png"),
      "image/png",
      "public,max-age=86400",
    ),
    uploadFile(
      bucket,
      `${destination.prefix}/assets/InterVariable.woff2`,
      path.join(s3ReportRoot, "assets", "InterVariable.woff2"),
      "font/woff2",
      "public,max-age=86400",
    ),
  ]);
  console.log(
    `Published immutable campaign ${campaign.campaignId} (${manifest.bundleDigest}) and ${history.campaigns.length} history record(s)`,
  );
}

if (
  process.argv[1] &&
  path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
) {
  await main().catch((error) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exitCode = 1;
  });
}