PaperClipAI/scripts/ci/refresh-test-durations.mjs

import assert from "node:assert/strict";
import { execFile, execFileSync } from "node:child_process";
import { readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify, stripVTControlCharacters } from "node:util";
import { timingPrefix } from "./test-timing-reporter.mjs";

const repository = "paperclipai/paperclip";
const api = `/repos/${repository}`;
const execute = promisify(execFile);
const github = async (endpoint, json = true) => {
  const { stdout } = await execute("gh", ["api", endpoint], {
    encoding: "utf8", maxBuffer: json ? 4 * 1024 * 1024 : 128 * 1024 * 1024, timeout: 120_000,
  });
  return json ? JSON.parse(stdout) : stdout;
};

export function validateRun(run) {
  assert.equal(run.repository?.full_name, repository);
  assert.equal(run.repository?.id, 1170821064);
  assert.equal(run.head_repository?.id, 1170821064);
  assert.equal(run.head_branch, "master");
  assert.equal(run.event, "push");
  assert.equal(run.path, ".github/workflows/cloud-readiness.yml");
  assert.equal(run.status, "completed");
  assert.equal(run.conclusion, "success");
  assert.match(run.head_sha, /^[a-f0-9]{40}$/);
  assert.ok(Number.isSafeInteger(run.id) && run.id > 0);
  assert.ok(Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0);
}

export function parseTimings(log, inventory) {
  const fallback = new Map(), measured = new Map();
  for (const raw of log.split("\n")) {
    const line = stripVTControlCharacters(raw);
    const offset = line.indexOf(timingPrefix);
    if (offset >= 0) {
      const record = JSON.parse(line.slice(offset + timingPrefix.length));
      assert.equal(record.version, 1);
      assert.equal(typeof record.file, "string");
      assert.match(record.file, /^server\/(src|scripts)\/.+\.test\.[cm]?[jt]s$/);
      assert.ok(!record.file.split("/").some(part => part === ".." || part === "." || part === ""));
      assert.ok(!record.file.includes("\\"));
      assert.ok(Number.isFinite(record.durationMs) && record.durationMs >= 0 && record.durationMs <= 3_600_000);
      // A trusted source run can predate a deletion or a lane reassignment.
      if (!inventory.has(record.file)) continue;
      measured.set(record.file, Math.max(measured.get(record.file) ?? 0, record.durationMs));
      continue;
    }
    // Bootstrap from pre-reporter runs. These values omit import/setup cost;
    // structured measurements take precedence as soon as master emits them.
    const match = line.match(/@paperclipai\/server\s+([^\s]+\.test\.[cm]?[jt]s)\s+\(\d+ tests?[^)]*\)\s+(\d+)ms\s*$/);
    if (!match) continue;
    const file = `server/${match[1]}`, ms = Number(match[2]);
    if (inventory.has(file) && ms <= 3_600_000) fallback.set(file, Math.max(fallback.get(file) ?? 0, ms));
  }
  return new Map([...fallback].map(([file, ms]) => [file, { ms, complete: false }])
    .concat([...measured].map(([file, ms]) => [file, { ms, complete: true }])));
}

export function updatedDurations(previous, inventory, samples, sources) {
  const measured = [...inventory].filter(file => samples.some(sample => sample.has(file)));
  assert.ok(measured.length >= inventory.size * 0.9, `Incomplete timing coverage: ${measured.length}/${inventory.size}`);
  const durations = {};
  let complete = 0;
  for (const file of [...inventory].sort()) {
    const values = samples.flatMap(sample => sample.has(file) ? [sample.get(file)] : []);
    const full = values.filter(value => value.complete);
    if (full.length) complete++;
    const ms = (full.length ? full : values).map(value => value.ms).sort((a, b) => a - b);
    if (!ms.length) {
      if (Number.isFinite(previous.durations[file]) && previous.durations[file] >= 0) durations[file] = previous.durations[file];
      continue;
    }
    const middle = Math.floor(ms.length / 2);
    // A small floor prevents clock resolution from giving a suite zero cost.
    durations[file] = Math.max(100, Math.ceil(ms.length % 2 ? ms[middle] : (ms[middle - 1] + ms[middle]) / 2));
  }
  return {
    $comment: "Generated by node scripts/ci/refresh-test-durations.mjs from successful protected-master Cloud readiness runs. Median per-file cost includes tests, hooks, collection, setup and environment preparation when the timing reporter is present; older logs bootstrap test/hook durations only. Missing files retain their prior weight or the partitioner's median fallback. Timing changes only partition work; they never authorize skipping a test.",
    unit: "ms", sources, coverage: { measured: measured.length, withSetup: complete, inventory: inventory.size }, durations,
  };
}

export async function refreshDurations(runIds = []) {
  const inventory = JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general",
    "--group", "general-server", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
    encoding: "utf8", env: { ...process.env, GITHUB_WORKFLOW: "PR", PAPERCLIP_CI_TEST_SELECTION: "" },
  }));
  const lanes = {
    general: { files: new Set(inventory.selectedGeneralServerSuites), path: "scripts/general-server-shard-durations.json", samples: [] },
    serialized: { files: new Set(inventory.selectedSerializedSuites), path: "scripts/serialized-shard-durations.json", samples: [] },
  };
  if (!runIds.length) {
    const listed = await github(`${api}/actions/workflows/cloud-readiness.yml/runs?branch=master&event=push&status=success&per_page=3`);
    runIds = listed.workflow_runs.map(run => run.id).slice(0, 2);
  }
  assert.ok(runIds.length > 0 && runIds.length <= 3, "Expected one to three successful master runs");
  const sources = [];
  for (const id of runIds) {
    assert.match(String(id), /^\d+$/);
    const run = await github(`${api}/actions/runs/${id}`);
    validateRun(run);
    execFileSync("git", ["merge-base", "--is-ancestor", run.head_sha, "HEAD"]);
    const jobs = [];
    for (let page = 1; ; page++) {
      const result = await github(`${api}/actions/runs/${id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
      jobs.push(...result.jobs);
      if (result.jobs.length < 100) break;
    }
    const samples = { general: new Map(), serialized: new Map() };
    const selected = jobs.map(job => ({ job, lane: /General tests \(server \(\d+\/\d+\)\)$/.test(job.name) ? "general"
      : /Serialized tests \(\d+\/\d+\)$/.test(job.name) ? "serialized" : null })).filter(item => item.lane);
    assert.ok(selected.length, "No server timing jobs found");
    // Keep API traffic and log buffers bounded. Logs are parsed data, never code.
    for (let start = 0; start < selected.length; start += 3) {
      await Promise.all(selected.slice(start, start + 3).map(async ({ job, lane }) => {
        assert.equal(job.run_id, run.id);
        assert.equal(job.conclusion, "success");
        const parsed = parseTimings(await github(`${api}/actions/jobs/${job.id}/logs`, false), lanes[lane].files);
        for (const [file, value] of parsed) {
          const previous = samples[lane].get(file);
          if (!previous || (value.complete && !previous.complete)
            || (value.complete === previous.complete && value.ms > previous.ms)) samples[lane].set(file, value);
        }
      }));
    }
    for (const lane of Object.keys(lanes)) lanes[lane].samples.push(samples[lane]);
    sources.push({ run: run.id, attempt: run.run_attempt, sha: run.head_sha, url: run.html_url });
  }
  // Validate both lanes before replacing either checked-in manifest.
  const updates = Object.values(lanes).map(lane => [lane.path,
    updatedDurations(JSON.parse(readFileSync(lane.path, "utf8")), lane.files, lane.samples, sources)]);
  for (const [file, data] of updates) {
    writeFileSync(file, JSON.stringify(data, null, 2) + "\n");
    console.log(`${file}: ${data.coverage.measured}/${data.coverage.inventory} measured; ${data.coverage.withSetup} include setup`);
  }
}

if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  refreshDurations(process.argv.slice(2)).catch(error => { console.error(error); process.exitCode = 1; });
}