PaperClipAI/scripts/ci/pr-run-durations.mjs

import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import path from "node:path";
import { stripVTControlCharacters } from "node:util";

// e2e and the Runner package lane never run on master. Their weights come from
// PR-workflow runs of PRs that merged into master, pinned to the final head SHA.
// Logs are parsed as data; nothing from a run is executed.
export const repository = "paperclipai/paperclip";
export const repositoryId = 1170821064;
const api = `/repos/${repository}`;

export const runnerPackageRoot = "packages/paperclip-runner";
export const e2eManifest = "scripts/e2e-shard-durations.json";
export const runnerManifest = `${runnerPackageRoot}/scripts/vitest-lane-durations.json`;

export function validatePull(pull) {
  assert.equal(pull.base?.ref, "master");
  assert.equal(pull.base?.repo?.id, repositoryId);
  assert.ok(pull.merged_at, `PR #${pull.number} is not merged`);
  assert.match(pull.head?.sha ?? "", /^[a-f0-9]{40}$/);
  assert.match(pull.merge_commit_sha ?? "", /^[a-f0-9]{40}$/);
}

export function validatePrRun(run, pull) {
  assert.equal(run.repository?.full_name, repository);
  assert.equal(run.repository?.id, repositoryId);
  assert.equal(run.event, "pull_request");
  assert.equal(run.path, ".github/workflows/pr.yml");
  assert.equal(run.status, "completed");
  assert.equal(run.conclusion, "success");
  // A run of an earlier push measures code that never merged.
  assert.equal(run.head_sha, pull.head.sha);
  assert.ok(Number.isSafeInteger(run.id) && run.id > 0);
  assert.ok(Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0);
}

export function e2eShardJob(name) {
  const match = /^ci \/ e2e shard \((\d+)\/(\d+)\)$/.exec(name);
  return match ? { index: Number(match[1]), count: Number(match[2]) } : null;
}

// Before #19646: "ci / Verify Paperclip Runner (vitest package)", one lane.
// Since #19646: "ci / Verify Paperclip Runner (vitest package (K/N))".
export function runnerPackageJob(name) {
  if (name === "ci / Verify Paperclip Runner (vitest package)") return { index: 1, count: 1 };
  const match = /^ci \/ Verify Paperclip Runner \((?:vitest )?package \((\d+)\/(\d+)\)\)$/.exec(name);
  return match ? { index: Number(match[1]), count: Number(match[2]) } : null;
}

// Returns the jobs only when they form one complete, successful set of shards.
export function completeShardSet(jobs, classify) {
  const selected = jobs.map(job => ({ job, shard: classify(job.name) })).filter(item => item.shard);
  if (!selected.length) return null;
  const count = selected[0].shard.count;
  const indexes = new Set(selected.map(item => item.shard.index));
  if (selected.some(item => item.shard.count !== count || item.job.conclusion !== "success")) return null;
  if (indexes.size !== selected.length || indexes.size !== count) return null;
  for (let index = 1; index <= count; index++) if (!indexes.has(index)) return null;
  return selected.map(item => item.job);
}

function splitLogLine(raw) {
  const line = stripVTControlCharacters(raw).replace(/\r$/, "");
  const match = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d+))?Z ?(.*)$/.exec(line);
  if (!match) return { time: null, text: line };
  const fraction = Number(`0.${match[2] ?? "0"}`);
  return { time: Date.parse(`${match[1]}Z`) + fraction * 1000, text: match[3] };
}

// A location outside a spec (e.g. smoke-lab.shared.ts) belongs to the shard's
// spec named after it. Several candidates are told apart by the suffix that the
// describe title repeats, e.g. "(P1-P4)" for smoke-lab-p1-p4.spec.ts.
export function attributeSpec(location, title, shardSpecs) {
  if (location.endsWith(".spec.ts")) {
    assert.ok(!shardSpecs || shardSpecs.includes(location), `${location} is not in this shard`);
    return location;
  }
  assert.ok(shardSpecs, `No shard spec list to attribute ${location}`);
  const directory = path.posix.dirname(location);
  const stem = path.posix.basename(location).replace(/(\.shared)?\.[cm]?[jt]sx?$/, "");
  const candidates = shardSpecs.filter(spec => path.posix.dirname(spec) === directory
    && path.posix.basename(spec).startsWith(`${stem}-`));
  if (candidates.length === 1) return candidates[0];
  const lower = title.toLowerCase();
  const named = candidates.filter(spec => lower.includes(path.posix.basename(spec).slice(stem.length + 1, -".spec.ts".length)));
  assert.equal(named.length, 1, `Cannot attribute ${location} (${title}) to one spec`);
  return named[0];
}

// The e2e lane runs one Playwright worker, so each gap between consecutive
// result lines is the finishing test's cost including fixtures and navigation.
// The first gap starts when Playwright reports "Running N tests".
export function parseE2eLog(log, inventory) {
  let shardSpecs = null, start = null, previous = null, expected = null, seen = 0;
  const totals = new Map();
  for (const raw of log.split("\n")) {
    const { time, text } = splitLogLine(raw);
    if (time === null || text.startsWith("[WebServer]")) continue;
    const specs = /^shard \d+\/\d+ specs: (.+)$/.exec(text);
    if (specs) {
      shardSpecs = specs[1].trim().split(/\s+/);
      continue;
    }
    const running = /^Running (\d+) tests? using (\d+) workers?\b/.exec(text);
    if (running) {
      assert.equal(start, null, "Expected one Playwright run per job");
      assert.equal(Number(running[2]), 1, "Gap attribution requires a single Playwright worker");
      expected = Number(running[1]);
      start = previous = time;
      continue;
    }
    if (start === null) continue;
    const result = /^\s*\S+\s+\d+\s+\[[^\]]+\]\s+›\s+(tests\/e2e\/[^\s:]+):\d+:\d+\s+›\s+(.*)$/.exec(text);
    if (!result) continue;
    assert.ok(!result[1].split("/").some(part => part === ".." || part === "."), result[1]);
    seen++;
    const spec = attributeSpec(result[1], result[2], shardSpecs);
    const gap = Math.max(0, time - previous);
    previous = time;
    totals.set(spec, (totals.get(spec) ?? 0) + gap);
  }
  assert.notEqual(start, null, "No Playwright run found");
  assert.equal(seen, expected, `Parsed ${seen} of ${expected} Playwright results`);
  const parsed = new Map();
  for (const [spec, ms] of totals) {
    assert.ok(ms <= 3_600_000, `${spec}: implausible ${ms}ms`);
    // Deleted or renamed specs still appear in older trusted runs.
    if (inventory.has(spec)) parsed.set(spec, { ms: Math.round(ms), complete: true });
  }
  return parsed;
}

// Vitest prints one line per file, relative to the package:
//  ✓ src/live/runnerd-codex-transport.test.ts (257 tests) 126679ms
// The value covers tests and hooks, not collection or setup.
export function parseRunnerLog(log, inventory) {
  const parsed = new Map();
  let expected = 0, seen = 0, summaries = 0;
  for (const raw of log.split("\n")) {
    const { text } = splitLogLine(raw);
    const summary = /^\s*Test Files\s+(?:\d+ failed \| )?(\d+) passed\b/.exec(text);
    if (summary) {
      summaries++;
      expected += Number(summary[1]);
      continue;
    }
    const match = /^\s*✓\s+(\S+\.test\.[cm]?[jt]sx?)\s+\(\d+ tests?[^)]*\)\s+(\d+)ms\s*$/.exec(text);
    if (!match) continue;
    assert.ok(!match[1].split("/").some(part => part === ".." || part === "." || part === ""), match[1]);
    assert.ok(!match[1].includes("\\"));
    seen++;
    const file = `${runnerPackageRoot}/${match[1]}`, ms = Number(match[2]);
    assert.ok(ms <= 3_600_000, `${file}: implausible ${ms}ms`);
    if (inventory.has(file)) parsed.set(file, { ms: Math.max(parsed.get(file)?.ms ?? 0, ms), complete: false });
  }
  assert.ok(summaries > 0, "No Vitest summary found");
  assert.equal(seen, expected, `Parsed ${seen} of ${expected} passed Vitest files`);
  return parsed;
}

// The Runner include globs, read as text so the refresh needs no installed
// dependencies. Tests assert the list stays non-empty and matches real files.
export function runnerIncludeGlobs(repoRoot = process.cwd()) {
  const config = readFileSync(path.join(repoRoot, runnerPackageRoot, "vitest.config.ts"), "utf8");
  const block = /\binclude:\s*\[([\s\S]*?)\]/.exec(config);
  assert.ok(block, "Runner vitest.config.ts has no include list");
  const globs = [...block[1].matchAll(/"([^"]+)"/g)].map(match => match[1]);
  assert.ok(globs.length > 0, "Runner include list is empty");
  return globs;
}

export function runnerInventory(repoRoot = process.cwd()) {
  const globs = runnerIncludeGlobs(repoRoot);
  const tracked = execFileSync("git", ["ls-files", "-z", "--", runnerPackageRoot], { cwd: repoRoot, encoding: "utf8" })
    .split("\0").filter(Boolean);
  return new Set(tracked.filter(file => {
    const relative = file.slice(runnerPackageRoot.length + 1);
    return globs.some(glob => path.matchesGlob(relative, glob));
  }));
}

export function isMergedIntoHead(sha, repoRoot = process.cwd()) {
  try {
    execFileSync("git", ["merge-base", "--is-ancestor", sha, "HEAD"], { cwd: repoRoot, stdio: "ignore" });
    return true;
  } catch {
    return false;
  }
}

function keepLargest(target, parsed) {
  for (const [file, value] of parsed) {
    const previous = target.get(file);
    if (!previous || value.ms > previous.ms) target.set(file, value);
  }
}

// lanes: [{ name, classify(jobName) -> {index,count}|null, parse(log, inventory), inventory }]
// Returns { [lane]: { samples: Map[], sources: [] } } with at most maxRuns samples each.
export async function collectMergedPrSamples({ gh, lanes, maxRuns = 3, pullLimit = 40,
  isMerged = isMergedIntoHead, log = console.log } = {}) {
  const result = Object.fromEntries(lanes.map(lane => [lane.name, { samples: [], sources: [] }]));
  const pulls = await gh(`${api}/pulls?state=closed&base=master&sort=updated&direction=desc&per_page=${pullLimit}`);
  const merged = pulls.filter(pull => pull.merged_at).sort((a, b) => b.merged_at.localeCompare(a.merged_at));
  for (const pull of merged) {
    const pending = lanes.filter(lane => result[lane.name].samples.length < maxRuns);
    if (!pending.length) break;
    validatePull(pull);
    // Only measure code present in this checkout.
    if (!isMerged(pull.merge_commit_sha)) continue;
    const listed = await gh(`${api}/actions/workflows/pr.yml/runs?head_sha=${pull.head.sha}&event=pull_request&status=success&per_page=20`);
    const runs = listed.workflow_runs.filter(run => {
      try { validatePrRun(run, pull); return true; } catch { return false; }
    }).sort((a, b) => b.id - a.id);
    if (!runs.length) continue;
    const run = runs[0];
    const jobs = [];
    for (let page = 1; ; page++) {
      const response = await gh(`${api}/actions/runs/${run.id}/jobs?filter=latest&per_page=100&page=${page}`);
      jobs.push(...response.jobs);
      if (response.jobs.length < 100) break;
    }
    for (const lane of pending) {
      const selected = completeShardSet(jobs, lane.classify);
      if (!selected) continue;
      const sample = new Map();
      try {
        for (const job of selected) {
          assert.equal(job.run_id, run.id);
          keepLargest(sample, lane.parse(await gh(`${api}/actions/jobs/${job.id}/logs`, false), lane.inventory));
        }
      } catch (error) {
        log(`::warning::${lane.name}: skipped run ${run.id} for PR #${pull.number}: ${error.message.split("\n")[0]}`);
        continue;
      }
      result[lane.name].samples.push(sample);
      result[lane.name].sources.push({ pull: pull.number, run: run.id, attempt: run.run_attempt, sha: run.head_sha, url: run.html_url });
    }
  }
  return result;
}