import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { stripVTControlCharacters } from "node:util"; import { listE2eSpecs } from "../e2e-shard.mjs"; import { ghApi } from "./gh-api.mjs"; import { oversizedThresholdEnv, oversizedThresholdMs } from "./oversized-test-files.mjs"; import { collectMergedPrSamples, e2eManifest, e2eShardJob, isMergedIntoHead, parseE2eLog, parseRunnerLog, runnerInventory, runnerManifest, runnerPackageJob } from "./pr-run-durations.mjs"; import { timingPrefix } from "./test-timing-reporter.mjs"; const repository = "paperclipai/paperclip"; const api = `/repos/${repository}`; const github = (endpoint, json = true) => ghApi(endpoint, { json }); const masterComment = "Generated by node scripts/ci/refresh-test-durations.mjs from successful protected-master Cloud readiness runs. Median per-file cost includes tests, hooks, collection, setup and environment preparation when the timing reporter is present; older logs bootstrap test/hook durations only. Missing files retain their prior weight or the partitioner's median fallback. Timing changes only partition work; they never authorize skipping a test."; const e2eComment = "Generated by node scripts/ci/refresh-test-durations.mjs from successful .github/workflows/pr.yml runs of PRs merged into master, each pinned to the PR's final head SHA. Per spec: the sum of gaps between consecutive Playwright result lines in 'Run e2e tests' (one worker), so fixture and navigation time count; tests located in a *.shared.ts file are attributed to the shard's matching spec. Median across runs. scripts/e2e-shard.mjs uses these weights only to place specs on shards; they never change which specs run. Missing specs retain their prior weight or the median fallback."; const generalComment = "Generated by node scripts/ci/refresh-test-durations.mjs from successful .github/workflows/pr.yml runs of PRs merged into master, each pinned to the PR's final head SHA and each covering at least 90% of the general server lane. Per file: the cost the timing reporter prints in the General tests (server) jobs, which includes tests, hooks, collection, setup and environment preparation under the lane's own file concurrency. Median across runs. Missing files retain their prior weight or the partitioner's median fallback. Timing changes only partition work; they never authorize skipping a test."; const runnerComment = "Generated by node scripts/ci/refresh-test-durations.mjs from successful .github/workflows/pr.yml runs of PRs merged into master, each pinned to the PR's final head SHA. Per file: the Vitest file duration printed by the Verify Paperclip Runner package job (tests and hooks; collection and setup are not included). Median across runs. Weights only place files on lanes; they never change which tests run. Missing files retain their prior weight or the median fallback."; export function validateRun(run) { assert.equal(run.repository?.full_name, repository); assert.equal(run.repository?.id, 1170821064); assert.equal(run.head_repository?.id, 1170821064); assert.equal(run.head_branch, "master"); assert.equal(run.event, "push"); assert.equal(run.path, ".github/workflows/cloud-readiness.yml"); assert.equal(run.status, "completed"); assert.equal(run.conclusion, "success"); assert.match(run.head_sha, /^[a-f0-9]{40}$/); assert.ok(Number.isSafeInteger(run.id) && run.id > 0); assert.ok(Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0); } export function parseTimings(log, inventory) { const fallback = new Map(), measured = new Map(); for (const raw of log.split("\n")) { const line = stripVTControlCharacters(raw); const offset = line.indexOf(timingPrefix); if (offset >= 0) { const record = JSON.parse(line.slice(offset + timingPrefix.length)); assert.equal(record.version, 1); assert.equal(typeof record.file, "string"); assert.match(record.file, /^server\/(src|scripts)\/.+\.test\.[cm]?[jt]s$/); assert.ok(!record.file.split("/").some(part => part === ".." || part === "." || part === "")); assert.ok(!record.file.includes("\\")); assert.ok(Number.isFinite(record.durationMs) && record.durationMs >= 0 && record.durationMs <= 3_600_000); // A trusted source run can predate a deletion or a lane reassignment. if (!inventory.has(record.file)) continue; measured.set(record.file, Math.max(measured.get(record.file) ?? 0, record.durationMs)); continue; } // Bootstrap from pre-reporter runs. These values omit import/setup cost; // structured measurements take precedence as soon as master emits them. const match = line.match(/@paperclipai\/server\s+([^\s]+\.test\.[cm]?[jt]s)\s+\(\d+ tests?[^)]*\)\s+(\d+)ms\s*$/); if (!match) continue; const file = `server/${match[1]}`, ms = Number(match[2]); if (inventory.has(file) && ms <= 3_600_000) fallback.set(file, Math.max(fallback.get(file) ?? 0, ms)); } return new Map([...fallback].map(([file, ms]) => [file, { ms, complete: false }]) .concat([...measured].map(([file, ms]) => [file, { ms, complete: true }]))); } export function updatedDurations(previous, inventory, samples, sources, { comment = masterComment } = {}) { const measured = [...inventory].filter(file => samples.some(sample => sample.has(file))); assert.ok(measured.length >= inventory.size * 0.9, `Incomplete timing coverage: ${measured.length}/${inventory.size}`); const durations = {}; let complete = 0; for (const file of [...inventory].sort()) { const values = samples.flatMap(sample => sample.has(file) ? [sample.get(file)] : []); const full = values.filter(value => value.complete); if (full.length) complete++; const ms = (full.length ? full : values).map(value => value.ms).sort((a, b) => a - b); if (!ms.length) { if (Number.isFinite(previous.durations[file]) && previous.durations[file] >= 0) durations[file] = previous.durations[file]; continue; } const middle = Math.floor(ms.length / 2); // A small floor prevents clock resolution from giving a suite zero cost. durations[file] = Math.max(100, Math.ceil(ms.length % 2 ? ms[middle] : (ms[middle - 1] + ms[middle]) / 2)); } return { $comment: comment, unit: "ms", sources, coverage: { measured: measured.length, withSetup: complete, inventory: inventory.size }, durations, }; } // Files over the per-file budget, slowest first, across every refreshed manifest. export function oversizedFiles(manifests, thresholdMs) { return manifests.flatMap(({ data }) => Object.entries(data.durations)) .filter(([, ms]) => ms > thresholdMs).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); } export const oversizedPlaceholder = ""; export function renderPrBody(template, oversized, thresholdMs) { assert.ok(template.includes(oversizedPlaceholder), "PR body template lacks the oversized-file placeholder"); const seconds = Math.round(thresholdMs / 1000); const lines = oversized.length ? [`- ${oversized.length} test file(s) exceed the ${seconds}s per-file budget (\`${oversizedThresholdEnv}\`). A shard cannot split a file, so each one bounds its lane's wall-clock time. Consider splitting:`, ...oversized.map(([file, ms]) => ` - \`${file}\`: ${Math.round(ms / 1000)}s`)] : [`- No measured test file exceeds the ${seconds}s per-file budget (\`${oversizedThresholdEnv}\`).`]; return template.replace(oversizedPlaceholder, lines.join("\n")); } function summarizeChange(file, previous, next) { const before = previous?.durations ?? {}, after = next.durations; const changes = Object.keys(after).map(key => [key, before[key], after[key]]) .filter(([, old, value]) => old !== value) .sort((a, b) => Math.abs(b[2] - (b[1] ?? 0)) - Math.abs(a[2] - (a[1] ?? 0))); const removed = Object.keys(before).filter(key => !(key in after)); console.log(`${file}: ${next.coverage.measured}/${next.coverage.inventory} measured; ${next.coverage.withSetup} include setup; ` + `${changes.length} changed, ${removed.length} removed; sources ${next.sources.map(source => source.pull ? `#${source.pull}/${source.run}` : source.run).join(", ")}`); for (const [key, old, value] of changes.slice(0, 12)) console.log(` ${key}: ${old ?? "(new)"} -> ${value}`); } export function parseArgs(argv) { const options = { runIds: [], dryRun: false, prBody: null, lanes: new Set(["server", "e2e", "runner"]) }; for (const arg of argv) { if (arg === "--dry-run") options.dryRun = true; else if (arg.startsWith("--pr-body=")) options.prBody = arg.slice("--pr-body=".length); else if (arg.startsWith("--lanes=")) { options.lanes = new Set(arg.slice("--lanes=".length).split(",")); for (const lane of options.lanes) assert.ok(["server", "e2e", "runner"].includes(lane), `Unknown lane ${lane}`); } else { assert.match(arg, /^\d+$/, `Unexpected argument ${arg}`); options.runIds.push(arg); } } return options; } export function generalServerShardCount(matrixFile = ".github/ci-task-matrix.json") { const entries = JSON.parse(readFileSync(matrixFile, "utf8")).general_tests .filter(entry => entry.group === "general-server-without-chat"); assert.ok(entries.length > 0 && entries.every(entry => entry.shard_count === entries.length)); return entries.length; } export function generalServerJob(name, fullCount = generalServerShardCount()) { const match = /^ci \/ General tests \(server \((\d+)\/(\d+)\)\)$/.exec(name); return match && Number(match[2]) === fullCount ? { index: Number(match[1]), count: Number(match[2]) } : null; } export const serverLane = name => /General tests \(server \(\d+\/\d+\)\)$/.test(name) ? "general" : /Serialized tests \(\d+\/\d+\)$/.test(name) ? "serialized" : null; const readManifest = file => existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : { durations: {} }; async function refreshMasterLanes(runIds) { const inventory = JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1", "--dry-run"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKFLOW: "PR", PAPERCLIP_CI_TEST_SELECTION: "" }, })); const lanes = { serialized: { files: new Set(inventory.selectedSerializedSuites), path: "scripts/serialized-shard-durations.json", samples: [] }, }; const runJobs = async run => { const jobs = []; for (let page = 1; ; page++) { const result = await github(`${api}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`); jobs.push(...result.jobs); if (result.jobs.length < 100) break; } return jobs; }; const candidates = []; if (runIds.length) { assert.ok(runIds.length <= 3, "Expected one to three successful master runs"); for (const id of runIds) { assert.match(String(id), /^\d+$/); const run = await github(`${api}/actions/runs/${id}`); validateRun(run); execFileSync("git", ["merge-base", "--is-ancestor", run.head_sha, "HEAD"]); candidates.push({ run, jobs: await runJobs(run) }); } } else { // Most master pushes only run Cloud admission and skip verification, and a // newer push can land after this checkout. Use the two newest runs this // checkout contains that actually ran the server test jobs. const listed = await github(`${api}/actions/workflows/cloud-readiness.yml/runs?branch=master&event=push&status=success&per_page=50`); for (const run of listed.workflow_runs) { if (candidates.length === 2) break; if (!isMergedIntoHead(run.head_sha)) continue; validateRun(run); const jobs = await runJobs(run); if (jobs.some(job => serverLane(job.name) === "serialized")) candidates.push({ run, jobs }); } } assert.ok(candidates.length > 0, "Expected one to three successful master runs"); const sources = []; for (const { run, jobs } of candidates) { const samples = { serialized: new Map() }; const selected = jobs.map(job => ({ job, lane: serverLane(job.name) })).filter(item => item.lane === "serialized"); assert.ok(selected.length, "No server timing jobs found"); // Keep API traffic and log buffers bounded. Logs are parsed data, never code. for (let start = 0; start < selected.length; start += 3) { await Promise.all(selected.slice(start, start + 3).map(async ({ job, lane }) => { assert.equal(job.run_id, run.id); assert.equal(job.conclusion, "success"); const parsed = parseTimings(await github(`${api}/actions/jobs/${job.id}/logs`, false), lanes[lane].files); for (const [file, value] of parsed) { const previous = samples[lane].get(file); if (!previous || (value.complete && !previous.complete) || (value.complete === previous.complete && value.ms > previous.ms)) samples[lane].set(file, value); } })); } for (const lane of Object.keys(lanes)) lanes[lane].samples.push(samples[lane]); sources.push({ run: run.id, attempt: run.run_attempt, sha: run.head_sha, url: run.html_url }); } return Object.values(lanes).map(lane => { const previous = readManifest(lane.path); return { path: lane.path, previous, data: updatedDurations(previous, lane.files, lane.samples, sources) }; }); } // Fewer than two valid runs leaves a manifest unchanged with a visible warning; // one run is too noisy to replace measured weights. export function generalServerInventory() { return new Set(JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1", "--dry-run"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKFLOW: "PR", PAPERCLIP_CI_TEST_SELECTION: "" }, })).selectedGeneralServerSuites); } async function refreshPrLanes(names) { if (!names.length) return []; const definitions = { general: () => { const inventory = generalServerInventory(); const fullCount = generalServerShardCount(); return { classify: name => generalServerJob(name, fullCount), parse: parseTimings, inventory, path: "scripts/general-server-shard-durations.json", comment: generalComment, maxRuns: 2, minFiles: Math.ceil(inventory.size * 0.9) }; }, e2e: () => ({ classify: e2eShardJob, parse: parseE2eLog, inventory: new Set(listE2eSpecs()), path: e2eManifest, comment: e2eComment }), runner: () => ({ classify: runnerPackageJob, parse: parseRunnerLog, inventory: runnerInventory(), path: runnerManifest, comment: runnerComment }), }; const lanes = names.map(name => ({ name, ...definitions[name]() })); const collected = await collectMergedPrSamples({ gh: github, lanes }); const updates = []; for (const lane of lanes) { const { samples, sources } = collected[lane.name]; if (samples.length < 2) { console.log(`::warning::${lane.path}: only ${samples.length} valid merged-PR run(s); keeping the current weights`); continue; } const previous = readManifest(lane.path); updates.push({ path: lane.path, previous, data: updatedDurations(previous, lane.inventory, samples, sources, { comment: lane.comment }) }); } return updates; } export async function refreshDurations(argv = []) { const options = parseArgs(argv.map(String)); const updates = [ ...(options.lanes.has("server") ? await refreshMasterLanes(options.runIds) : []), ...await refreshPrLanes([...(options.lanes.has("server") ? ["general"] : []), ...["e2e", "runner"].filter(lane => options.lanes.has(lane))]), ]; // Validate every lane before replacing any checked-in manifest. for (const { path: file, previous, data } of updates) { summarizeChange(file, previous, data); if (!options.dryRun) writeFileSync(file, JSON.stringify(data, null, 2) + "\n"); } const thresholdMs = oversizedThresholdMs(); const oversized = oversizedFiles(updates, thresholdMs); console.log(`${oversized.length} measured test file(s) exceed ${Math.round(thresholdMs / 1000)}s`); for (const [file, ms] of oversized) console.log(` ${file}: ${Math.round(ms / 1000)}s`); if (options.prBody) { writeFileSync(options.prBody, renderPrBody(readFileSync(".github/ci-duration-pr.md", "utf8"), oversized, thresholdMs)); } if (options.dryRun) console.log("Dry run: no manifest was written"); return updates; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { refreshDurations(process.argv.slice(2)).catch(error => { console.error(error); process.exitCode = 1; }); }