import assert from "node:assert/strict"; import { execFile, execFileSync } from "node:child_process"; import { readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify, stripVTControlCharacters } from "node:util"; import { timingPrefix } from "./test-timing-reporter.mjs"; const repository = "paperclipai/paperclip"; const api = `/repos/${repository}`; const execute = promisify(execFile); const github = async (endpoint, json = true) => { const { stdout } = await execute("gh", ["api", endpoint], { encoding: "utf8", maxBuffer: json ? 4 * 1024 * 1024 : 128 * 1024 * 1024, timeout: 120_000, }); return json ? JSON.parse(stdout) : stdout; }; export function validateRun(run) { assert.equal(run.repository?.full_name, repository); assert.equal(run.repository?.id, 1170821064); assert.equal(run.head_repository?.id, 1170821064); assert.equal(run.head_branch, "master"); assert.equal(run.event, "push"); assert.equal(run.path, ".github/workflows/cloud-readiness.yml"); assert.equal(run.status, "completed"); assert.equal(run.conclusion, "success"); assert.match(run.head_sha, /^[a-f0-9]{40}$/); assert.ok(Number.isSafeInteger(run.id) && run.id > 0); assert.ok(Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0); } export function parseTimings(log, inventory) { const fallback = new Map(), measured = new Map(); for (const raw of log.split("\n")) { const line = stripVTControlCharacters(raw); const offset = line.indexOf(timingPrefix); if (offset >= 0) { const record = JSON.parse(line.slice(offset + timingPrefix.length)); assert.equal(record.version, 1); assert.equal(typeof record.file, "string"); assert.match(record.file, /^server\/(src|scripts)\/.+\.test\.[cm]?[jt]s$/); assert.ok(!record.file.split("/").some(part => part === ".." || part === "." || part === "")); assert.ok(!record.file.includes("\\")); assert.ok(Number.isFinite(record.durationMs) && record.durationMs >= 0 && record.durationMs <= 3_600_000); // A trusted source run can predate a deletion or a lane reassignment. if (!inventory.has(record.file)) continue; measured.set(record.file, Math.max(measured.get(record.file) ?? 0, record.durationMs)); continue; } // Bootstrap from pre-reporter runs. These values omit import/setup cost; // structured measurements take precedence as soon as master emits them. const match = line.match(/@paperclipai\/server\s+([^\s]+\.test\.[cm]?[jt]s)\s+\(\d+ tests?[^)]*\)\s+(\d+)ms\s*$/); if (!match) continue; const file = `server/${match[1]}`, ms = Number(match[2]); if (inventory.has(file) && ms <= 3_600_000) fallback.set(file, Math.max(fallback.get(file) ?? 0, ms)); } return new Map([...fallback].map(([file, ms]) => [file, { ms, complete: false }]) .concat([...measured].map(([file, ms]) => [file, { ms, complete: true }]))); } export function updatedDurations(previous, inventory, samples, sources) { const measured = [...inventory].filter(file => samples.some(sample => sample.has(file))); assert.ok(measured.length >= inventory.size * 0.9, `Incomplete timing coverage: ${measured.length}/${inventory.size}`); const durations = {}; let complete = 0; for (const file of [...inventory].sort()) { const values = samples.flatMap(sample => sample.has(file) ? [sample.get(file)] : []); const full = values.filter(value => value.complete); if (full.length) complete++; const ms = (full.length ? full : values).map(value => value.ms).sort((a, b) => a - b); if (!ms.length) { if (Number.isFinite(previous.durations[file]) && previous.durations[file] >= 0) durations[file] = previous.durations[file]; continue; } const middle = Math.floor(ms.length / 2); // A small floor prevents clock resolution from giving a suite zero cost. durations[file] = Math.max(100, Math.ceil(ms.length % 2 ? ms[middle] : (ms[middle - 1] + ms[middle]) / 2)); } return { $comment: "Generated by node scripts/ci/refresh-test-durations.mjs from successful protected-master Cloud readiness runs. Median per-file cost includes tests, hooks, collection, setup and environment preparation when the timing reporter is present; older logs bootstrap test/hook durations only. Missing files retain their prior weight or the partitioner's median fallback. Timing changes only partition work; they never authorize skipping a test.", unit: "ms", sources, coverage: { measured: measured.length, withSetup: complete, inventory: inventory.size }, durations, }; } export async function refreshDurations(runIds = []) { const inventory = JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1", "--dry-run"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKFLOW: "PR", PAPERCLIP_CI_TEST_SELECTION: "" }, })); const lanes = { general: { files: new Set(inventory.selectedGeneralServerSuites), path: "scripts/general-server-shard-durations.json", samples: [] }, serialized: { files: new Set(inventory.selectedSerializedSuites), path: "scripts/serialized-shard-durations.json", samples: [] }, }; if (!runIds.length) { const listed = await github(`${api}/actions/workflows/cloud-readiness.yml/runs?branch=master&event=push&status=success&per_page=3`); runIds = listed.workflow_runs.map(run => run.id).slice(0, 2); } assert.ok(runIds.length > 0 && runIds.length <= 3, "Expected one to three successful master runs"); const sources = []; for (const id of runIds) { assert.match(String(id), /^\d+$/); const run = await github(`${api}/actions/runs/${id}`); validateRun(run); execFileSync("git", ["merge-base", "--is-ancestor", run.head_sha, "HEAD"]); const jobs = []; for (let page = 1; ; page++) { const result = await github(`${api}/actions/runs/${id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`); jobs.push(...result.jobs); if (result.jobs.length < 100) break; } const samples = { general: new Map(), serialized: new Map() }; const selected = jobs.map(job => ({ job, lane: /General tests \(server \(\d+\/\d+\)\)$/.test(job.name) ? "general" : /Serialized tests \(\d+\/\d+\)$/.test(job.name) ? "serialized" : null })).filter(item => item.lane); assert.ok(selected.length, "No server timing jobs found"); // Keep API traffic and log buffers bounded. Logs are parsed data, never code. for (let start = 0; start < selected.length; start += 3) { await Promise.all(selected.slice(start, start + 3).map(async ({ job, lane }) => { assert.equal(job.run_id, run.id); assert.equal(job.conclusion, "success"); const parsed = parseTimings(await github(`${api}/actions/jobs/${job.id}/logs`, false), lanes[lane].files); for (const [file, value] of parsed) { const previous = samples[lane].get(file); if (!previous || (value.complete && !previous.complete) || (value.complete === previous.complete && value.ms > previous.ms)) samples[lane].set(file, value); } })); } for (const lane of Object.keys(lanes)) lanes[lane].samples.push(samples[lane]); sources.push({ run: run.id, attempt: run.run_attempt, sha: run.head_sha, url: run.html_url }); } // Validate both lanes before replacing either checked-in manifest. const updates = Object.values(lanes).map(lane => [lane.path, updatedDurations(JSON.parse(readFileSync(lane.path, "utf8")), lane.files, lane.samples, sources)]); for (const [file, data] of updates) { writeFileSync(file, JSON.stringify(data, null, 2) + "\n"); console.log(`${file}: ${data.coverage.measured}/${data.coverage.inventory} measured; ${data.coverage.withSetup} include setup`); } } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { refreshDurations(process.argv.slice(2)).catch(error => { console.error(error); process.exitCode = 1; }); }