diff --git a/.github/scripts/tests/ci-profile.test.mjs b/.github/scripts/tests/ci-profile.test.mjs index 39f21452fa7..e1845face70 100644 --- a/.github/scripts/tests/ci-profile.test.mjs +++ b/.github/scripts/tests/ci-profile.test.mjs @@ -5,9 +5,9 @@ import { execFileSync, spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { buildTrace } from "../../../scripts/ci-profile/trace.mjs"; -import { sendEvents } from "../../../scripts/ci-profile/export.mjs"; -import { associatedProfilePr, ciOwners, trustedOwnerIds } from "../../../scripts/ci-profile/authorize.mjs"; +import { buildTrace, normalizeChecks, pendingChecks, traceUrl } from "../../../scripts/ci-profile/trace.mjs"; +import { sendEvents, settledChecks } from "../../../scripts/ci-profile/export.mjs"; +import { associatedOwnerPr, associatedProfilePr, ciOwners, exportMode, sampleRate, trustedOwnerIds } from "../../../scripts/ci-profile/authorize.mjs"; const at = (seconds) => new Date(1_800_000_000_000 + seconds * 1000).toISOString(); const run = { @@ -186,6 +186,136 @@ test("label alone cannot authorize outsiders, bots, stale revisions or outsider assert.equal(associatedProfilePr(run, [pr], []), undefined); }); +test("always-on export samples whole CODEOWNER runs and never needs the label", () => { + const pr = { user: { id: 123, type: "User" }, base: { repo: run.repository }, head: { repo: { id: 99 }, sha: "abc" }, labels: [] }; + assert.equal(sampleRate, 10); + assert.equal(exportMode({ ...run, run_number: 64800, display_title: "fix: something" }), "sampled"); + for (const number of [64801, 64805, 64809, 0, undefined, "64800"]) { + assert.equal(exportMode({ ...run, run_number: number, display_title: "fix: something" }), undefined); + } + assert.equal(exportMode({ ...run, run_number: 64801, display_title: "CI profile · PR #5" }), "label"); + assert.equal(associatedOwnerPr(run, [pr], [123]), pr); + assert.equal(associatedProfilePr(run, [pr], [123]), undefined); + for (const changed of [ + { ...pr, user: { id: 999, type: "User" } }, + { ...pr, user: { id: 123, type: "Bot" } }, + { ...pr, head: { ...pr.head, sha: "next" } }, + { ...pr, head: { ...pr.head, repo: { id: 100 } } }, + { ...pr, head: { ...pr.head, repo: null } }, + { ...pr, base: { repo: { ...run.repository, id: 100 } } }, + ]) assert.equal(associatedOwnerPr(run, [changed], [123]), undefined); + assert.equal(associatedOwnerPr({ ...run, actor: { id: 999, type: "User" } }, [pr], [123]), undefined); + assert.equal(associatedOwnerPr({ ...run, triggering_actor: { id: 41898282, type: "Bot" } }, [pr], [123]), undefined); + assert.equal(associatedOwnerPr(run, [pr], []), undefined); + + const trace = buildTrace({ ...run, conclusion: "success" }, jobs, [], { sampleRate, testDetail: false }); + assert.deepEqual([...new Set(trace.events.map((event) => event.samplerate))], [10]); + assert.equal(trace.events[0].data["ci.sample_rate"], 10); + assert.deepEqual(trace.warnings, ["Job/step timings only; no test reporter collection steps were present"]); + assert.equal(buildTrace(run, jobs).events[0].samplerate, 1); + for (const invalid of [0, 1.5, -1, "10"]) assert.throws(() => buildTrace(run, jobs, [], { sampleRate: invalid }), /sample rate/); +}); + +test("outside checks are spans on the run's clock, with elapsed time to the last one", () => { + const head = { ...run, conclusion: "success", path: ".github/workflows/pr.yml" }; + const checks = normalizeChecks(head, { + checkRuns: [ + { app: { slug: "greptile-apps" }, name: "Greptile Review", status: "completed", conclusion: "success", started_at: at(1), completed_at: at(90), output: { text: "must not leave GitHub" } }, + { app: { slug: "github-actions" }, name: "ci / first", status: "completed", conclusion: "success", started_at: at(2), completed_at: at(12) }, + { app: { slug: "scanner" }, name: "Scan", status: "in_progress", conclusion: null, started_at: at(1), completed_at: null }, + { app: { slug: "scanner" }, name: "Skipped", status: "completed", conclusion: "skipped", started_at: at(9), completed_at: at(1) }, + { app: { slug: "old" }, name: "Before this run", status: "completed", conclusion: "success", started_at: at(-600), completed_at: at(-500) }, + { name: "no app", status: "completed", started_at: at(1), completed_at: at(2) }, + ], + statuses: [ + { context: "security/scan", state: "success", created_at: at(15) }, + { context: "security/scan", state: "pending", created_at: at(0) }, + { context: "slow/status", state: "pending", created_at: at(0) }, + { context: "broken/status", state: "error", created_at: at(4) }, + { context: "broken/status", state: "pending", created_at: at(1) }, + ], + workflowRuns: [ + { id: 42, path: ".github/workflows/pr.yml", head_sha: "abc", event: "pull_request", name: "PR", status: "completed", created_at: at(0), updated_at: at(20) }, + { id: 50, path: ".github/workflows/review.yml", head_sha: "abc", event: "pull_request_target", name: "Review", status: "completed", conclusion: "success", run_attempt: 1, created_at: at(0), updated_at: at(40) }, + { id: 51, path: ".github/workflows/review.yml", head_sha: "abc", event: "pull_request_target", name: "Review", status: "completed", conclusion: "failure", run_attempt: 1, created_at: at(5), updated_at: at(60) }, + { id: 52, path: ".github/workflows/other.yml", head_sha: "other", event: "pull_request", name: "Other commit", status: "completed", created_at: at(0), updated_at: at(9) }, + { id: 53, path: ".github/workflows/ci-profile.yml", head_sha: "abc", event: "workflow_run", name: "CI profile", status: "completed", created_at: at(0), updated_at: at(9) }, + ], + }); + assert.deepEqual(checks.map((check) => check.name).sort(), + ["Before this run", "Greptile Review", "Review", "Scan", "Skipped", "broken/status", "security/scan", "slow/status"]); + assert.deepEqual(pendingChecks(head, checks).map((check) => check.name), ["Scan", "slow/status"]); + + const trace = buildTrace(head, jobs, [], { checks }); + const spans = trace.events.filter((event) => /_check$/.test(event.data["ci.kind"])); + assert.deepEqual(spans.map((event) => [event.data.name, event.data["ci.kind"], event.data.duration_ms, event.data["ci.status"]]), [ + ["Greptile Review", "external_check", 89000, "success"], + ["security/scan", "external_check", 15000, "success"], + ["broken/status", "external_check", 3000, "error"], + ["Review", "workflow_check", 55000, "failure"], + ]); + assert.deepEqual(spans.map((event) => event.data.error), [false, false, true, true]); + const root = trace.events[0].data; + assert.ok(spans.every((event) => event.data["trace.parent_id"] === root["trace.span_id"])); + assert.equal(spans[0].data["ci.finishes_after_workflow"], true); + assert.equal(spans[1].data["ci.finishes_after_workflow"], false); + assert.equal(root.duration_ms, 20000); + assert.equal(root["ci.workflow_ms"], 20000); + assert.equal(root["ci.all_checks_ms"], 90000); + assert.equal(root["ci.last_to_finish"], "Greptile Review"); + assert.equal(root["ci.slowest_check"], "Greptile Review"); + assert.equal(root["ci.slowest_check_ms"], 89000); + assert.equal(root["ci.check_count"], 4); + assert.equal(root["ci.pending_check_count"], 2); + assert.equal(root["ci.pending_checks"], "Scan, slow/status"); + assert.equal(root["ci.earlier_check_count"], 1); + assert.equal(root["ci.profile_complete"], false); + assert.match(trace.warnings.join(), /2 outside checks were still running/); + assert.doesNotMatch(JSON.stringify(trace), /must not leave GitHub/); + assert.match(traceUrl("team", "test", "paperclip-ci", trace), /trace_end_ts=1800000090$/); + + const alone = buildTrace(head, jobs).events[0].data; + assert.equal(alone["ci.all_checks_ms"], 20000); + assert.equal(alone["ci.last_to_finish"], "PR workflow"); + assert.equal("ci.slowest_check" in alone, false); + // A rerun repeats jobs only; the push's outside checks are not its elapsed time. + const rerun = buildTrace({ ...head, run_attempt: 2, run_started_at: at(0) }, jobs.map((entry) => ({ ...entry, run_attempt: 2 })), [], { checks }); + assert.equal(rerun.events[0].data["ci.check_count"], 0); + assert.equal(rerun.events[0].data["ci.all_checks_ms"], 20000); +}); + +test("export waits a bounded time for outside checks, never for other workflows", async () => { + const head = { ...run, path: ".github/workflows/pr.yml" }; + const check = (completed) => ({ kind: "external_check", app: "greptile-apps", name: "Greptile Review", startedAt: at(1), completedAt: completed ? at(90) : null }); + const pauses = []; + let clock = 0; + const now = () => clock; + const pause = async (ms) => { pauses.push(ms); clock += ms; }; + let loads = 0; + const settled = await settledChecks(head, () => [check(++loads >= 3)], { pause, now }); + assert.equal(settled[0].completedAt, at(90)); + assert.deepEqual(pauses, [30_000, 30_000]); + + pauses.length = 0; + const stuck = await settledChecks(head, () => [check(false)], { pause, now, waitMs: 90_000 }); + assert.equal(stuck[0].completedAt, null); + assert.deepEqual(pauses, [30_000, 30_000, 30_000]); + + // Slow GitHub responses count against the same deadline as the pauses. + pauses.length = 0; + loads = 0; + await settledChecks(head, () => { clock += 20_000; loads += 1; return [check(false)]; }, { pause, now, waitMs: 90_000 }); + assert.deepEqual(pauses, [30_000, 20_000]); + assert.equal(loads, 3); + + pauses.length = 0; + await settledChecks(head, () => [{ ...check(false), kind: "workflow_check" }], { pause, now }); + await settledChecks(head, () => [{ ...check(false), startedAt: at(-600) }], { pause, now }); + await settledChecks({ ...head, run_attempt: 2, run_started_at: at(0) }, () => [check(false)], { pause, now }); + await settledChecks(head, () => [check(false)], { pause, now, waitMs: 0 }); + assert.deepEqual(pauses, []); +}); + test("profiling uses only infrastructure CODEOWNERS resolved at trusted master", () => { const source = '.github/** @owner @second\ndoc/** @outsider\n'; assert.deepEqual(ciOwners(source), ["owner", "second"]); @@ -295,6 +425,21 @@ test("workflow exporter runs trusted code and keeps keys out of the PR jobs", () const workflow = readFileSync(new URL("../../workflows/ci-profile.yml", import.meta.url), "utf8"); assert.match(workflow, /ref: \$\{\{ github.sha \}\}/); assert.match(workflow, /startsWith\(github.event.workflow_run.display_title/); + // The job condition keeps outside contributors' runs from starting a runner + // or entering the environment, and matches the exporter's sample rule. + const condition = workflow.split(" if: >-\n")[1].split("\n runs-on:")[0]; + for (const required of [ + "github.repository_id == '1170821064'", + "github.event.workflow_run.event == 'pull_request'", + "github.event.workflow_run.path == '.github/workflows/pr.yml'", + "github.event.workflow_run.head_repository.full_name == 'paperclipai/paperclip'", + "contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.workflow_run.actor.id))", + "contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.workflow_run.triggering_actor.id))", + "endsWith(format('{0}', github.event.workflow_run.run_number), '0')", + ]) assert.ok(condition.includes(required), required); + assert.equal(condition.match(/\|\|/g).length, 1, "only label-or-sample may be an alternative"); + assert.equal(workflow.match(/^ [a-z_]+:$/gm).filter((line) => ![" workflow_run:", " export:"].includes(line)).length, 0); + assert.equal(workflow.match(/environment:/g).length, 1); assert.match(workflow, /environment: ci-profiling/); assert.doesNotMatch(workflow, /head_sha|secrets: inherit|contents: write|pull-requests: write|pnpm install/); const pr = readFileSync(new URL("../../workflows/pr-trusted.yml", import.meta.url), "utf8"); diff --git a/.github/workflows/ci-profile.yml b/.github/workflows/ci-profile.yml index 9034d543c0e..7f3ad50c2d0 100644 --- a/.github/workflows/ci-profile.yml +++ b/.github/workflows/ci-profile.yml @@ -7,17 +7,36 @@ on: permissions: actions: read + # Check runs and commit statuses time the outside reviews and scans. + checks: read contents: read pull-requests: read + statuses: read jobs: export: + # No runner starts, and the environment is never entered, unless all of + # these hold: + # - The run is the main PR workflow on a branch of this repository. + # - Its actor and rerun actor are on the maintainer roster that CI routing + # already uses. The exporter then verifies them and the PR author against + # master's CODEOWNERS before it reads or sends anything. + # - The run carries the profile-ci label, or its run number ends in 0. That + # is the one-in-ten sample; scripts/ci-profile/authorize.mjs applies the + # same rule. if: >- github.repository == 'paperclipai/paperclip' && + github.repository_id == '1170821064' && github.event.workflow_run.event == 'pull_request' && - startsWith(github.event.workflow_run.display_title, 'CI profile · PR #') + github.event.workflow_run.path == '.github/workflows/pr.yml' && + github.event.workflow_run.head_repository.full_name == 'paperclipai/paperclip' && + contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.workflow_run.actor.id)) && + contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.workflow_run.triggering_actor.id)) && + (startsWith(github.event.workflow_run.display_title, 'CI profile · PR #') || + endsWith(format('{0}', github.event.workflow_run.run_number), '0')) runs-on: ubuntu-latest - timeout-minutes: 10 + # Covers the bounded wait for outside checks that outlast the workflow. + timeout-minutes: 30 # This environment permits only the master branch. Do not use a repository # secret: an in-repository PR must not be able to request this credential. environment: ci-profiling diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 41af89a05ca..f17b66adec9 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -462,7 +462,8 @@ Changing that file selects its consumers. Generated/untracked files and unknown query loaders retain their consumers conservatively. Other filesystem, process and network effects keep their existing fallback. -For a full wall-clock profile of all main PR CI jobs, add `profile-ci` to the PR. +One in ten CI CODEOWNER PR runs sends its job, step and outside-check timings to +Honeycomb. For a full wall-clock profile of all main PR CI jobs, add `profile-ci` to the PR. See [CI profiling](ci-profiling.md) for the Honeycomb waterfall, coverage, operator setup, and activation requirements. diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index 9975cff7d3b..307830f76b1 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -1,4 +1,53 @@ -# Profile the complete PR suite +# CI timing in Honeycomb + +CI CODEOWNER PRs send their CI timing to Honeycomb in two ways: + +| | Sampled (always on) | `profile-ci` label | +| --- | --- | --- | +| Which runs | One `PR` run in ten, by run number | Every run while the label is on | +| CI scope | Whatever the PR normally runs | All lanes of the main PR suite | +| Trace detail | Workflow, jobs, steps, outside checks | The same, plus test files and tests | +| Sample rate on spans | 10 | 1 | + +Both are exported by the **CI profile** workflow after the `PR` run finishes, +to the `paperclip-ci` dataset. Outside contributors' PRs are never exported. + +## Sampled runs + +A `PR` run is sampled when its run number ends in 0. Sampling keeps or drops a +whole run: a sampled trace has every job and step. Each span carries a sample +rate of 10, so Honeycomb's counts and sums scale back to all runs, and duration +percentiles need no correction. `ci.profile_source` is `sampled`, and +`ci.pr_number` identifies the PR. Sampled runs change nothing in the PR jobs +and write no timing artifacts; job and step times come from the GitHub API. + +A particular PR's run is in Honeycomb only if it was sampled. To inspect one +PR, add the label. + +## Outside checks and total time + +Every trace also has one span for each check on the PR's head commit that is +not a job of the `PR` workflow: + +- `ci.kind=external_check`: reviews and scans from other apps, such as + Greptile, and commit statuses (`ci.check_app=commit-status`). +- `ci.kind=workflow_check`: other workflows the PR started, one span each. + +The root span has two totals. `ci.workflow_ms` is the `PR` workflow, from run +creation to its last job. `ci.all_checks_ms` runs from the same start to the +last check of any kind, and `ci.last_to_finish` names it. `ci.slowest_check` +and `ci.slowest_check_ms` name the longest outside check. + +If an outside app's check is still running when the `PR` workflow finishes, +the exporter waits up to 15 minutes for it. It does not wait for other +workflows. Checks still running at export have no span; they are counted in +`ci.pending_check_count` and listed in `ci.pending_checks`, and +`ci.all_checks_ms` excludes them. Checks that started more than a minute +before the run, such as those from a push before a later label event, are +counted in `ci.earlier_check_count` and left out. Reruns carry no outside +checks. A workflow's completion time is its last update in GitHub. + +## Profile the complete PR suite Add the `profile-ci` label to a PR authored by a CI CODEOWNER. Adding the label starts a fresh `PR` workflow run. Further pushes while the label remains also run with profiling enabled. @@ -20,8 +69,9 @@ A label on an outside contributor's PR does not enable profiling, even when a maintainer applies it. The PR still runs normal CI and needs no Honeycomb access. Failed identity lookups disable profiling. -Ordinary PRs do not write timing artifacts or export traces. The exporter job is -skipped for ordinary run names. Adding another label does not run the CI matrix. +Ordinary PRs do not write timing artifacts. The exporter job is skipped for +ordinary run names unless the run is sampled. Adding another label does not run +the CI matrix. Profiling preserves the existing trusted AWS runner routing. Eligible PR jobs use `paperclip-public-pr-x64`. The synchronized maintainer admission roster @@ -40,8 +90,8 @@ cannot be authorized. The separate post-run Honeycomb exporter also uses The workflow root measures elapsed time from run creation to the last job's completion. For reruns, it starts at that attempt's `run_started_at` instead of the original creation time. Every job is a child on the same wall clock, with -its steps and available test details beneath it. All spans use a sample rate of -one. There is no sampling within the selected attempt. For a compact Gantt view, +its steps and available test details beneath it. Labelled runs use a sample +rate of one. There is no sampling within the selected attempt. For a compact Gantt view, open a job row's menu and choose **Collapse spans at this depth**. Expand an individual job to inspect its steps and test files. @@ -129,8 +179,16 @@ Create the repository label `profile-ci` if it does not exist. the exporter, installs no dependencies, and parses PR artifacts as bounded JSON without extracting or executing their contents. The PR jobs never receive the Honeycomb key. The exporter independently verifies CI CODEOWNER identities, -the current label, repository, and exact tested PR revision before reading -artifacts or sending data. Its credential is sent only in an HTTPS header to +the repository, the exact tested PR revision and, for labelled runs, the +current label before reading artifacts or sending data. + +The export job's condition is a first filter, evaluated before a runner starts +or the environment is entered. It requires the main `PR` workflow on a branch +of this repository, an actor and rerun actor on the `AWS_CI_ROUTING_USER_IDS` +maintainer roster that CI routing uses, and a labelled or sampled run. A fork +PR, or a run started or rerun by anyone off the roster, skips the job. The +roster only filters: the CODEOWNERS check in the exporter decides. If the +roster variable is empty, nothing is exported. Its credential is sent only in an HTTPS header to `api.honeycomb.io`; redirects are rejected. Exported fields are allowlisted. `CODEOWNERS` also covers `scripts/ci-profile/**`, so changes to the privileged exporter require infrastructure-owner review. diff --git a/scripts/ci-profile/authorize.mjs b/scripts/ci-profile/authorize.mjs index f863e22d65e..1c9fc1cd7af 100644 --- a/scripts/ci-profile/authorize.mjs +++ b/scripts/ci-profile/authorize.mjs @@ -41,12 +41,29 @@ export function authorizedActors(run, ownerIds) { && allowed(run.actor) && allowed(run.triggering_actor); } -export function associatedProfilePr(run, pulls, ownerIds) { +// The PR at this exact revision, when its author, the run's actor and the +// rerun actor are all CI CODEOWNERS. +export function associatedOwnerPr(run, pulls, ownerIds, accept = () => true) { if (!authorizedActors(run, ownerIds)) return undefined; return pulls.find((pr) => pr.base.repo.id === repositoryId && pr.base.repo.full_name === repository && pr.head.repo?.id === run.head_repository?.id && pr.head.sha === run.head_sha - && pr.user?.type === "User" && ownerIds.includes(pr.user.id) - && pr.labels.some((label) => label.name === "profile-ci")); + && pr.user?.type === "User" && ownerIds.includes(pr.user.id) && accept(pr)); +} + +export function associatedProfilePr(run, pulls, ownerIds) { + return associatedOwnerPr(run, pulls, ownerIds, (pr) => pr.labels.some((label) => label.name === "profile-ci")); +} + +// Always-on export keeps one run in this many, chosen by the workflow's +// sequential run number so that ci-profile.yml can skip the others before +// starting a runner. +export const sampleRate = 10; + +// "label": a profile-ci run with test detail, always exported. +// "sampled": an ordinary run chosen by its number, job and step timings only. +export function exportMode(run) { + if (typeof run.display_title === "string" && run.display_title.startsWith("CI profile · PR #")) return "label"; + return Number.isSafeInteger(run.run_number) && run.run_number > 0 && run.run_number % sampleRate === 0 ? "sampled" : undefined; } async function main() { diff --git a/scripts/ci-profile/export.mjs b/scripts/ci-profile/export.mjs index 2e5ab2ba2e5..33d8fbf363f 100644 --- a/scripts/ci-profile/export.mjs +++ b/scripts/ci-profile/export.mjs @@ -3,8 +3,8 @@ import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { buildTrace, traceUrl } from "./trace.mjs"; -import { associatedProfilePr, authorizedActors, trustedOwnerIds } from "./authorize.mjs"; +import { buildTrace, normalizeChecks, pendingChecks, traceUrl } from "./trace.mjs"; +import { associatedOwnerPr, associatedProfilePr, authorizedActors, exportMode, sampleRate, trustedOwnerIds } from "./authorize.mjs"; const repository = "paperclipai/paperclip"; const repoPath = `/repos/${repository}`; @@ -46,6 +46,21 @@ export async function sendEvents(events, apiKey, dataset, fetcher = fetch, pause } } +// Reviews and scans from outside apps can outlast the workflow. Wait a bounded +// time so the trace records how long they took; whatever is still running +// after that is reported as pending instead of holding the export open. +export async function settledChecks(run, load, { waitMs = 15 * 60_000, intervalMs = 30_000, + pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), now = Date.now } = {}) { + // The deadline is on the wall clock, so slow GitHub responses count against it. + const deadline = now() + waitMs; + let checks = load(); + while (now() < deadline && pendingChecks(run, checks).length) { + await pause(Math.min(intervalMs, deadline - now())); + checks = load(); + } + return checks; +} + async function main() { const args = process.argv.slice(2); const arg = (name) => args.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); @@ -76,20 +91,31 @@ async function main() { return; } } - const pr = historical ? undefined : associatedProfilePr(run, pulls, ownerIds); - if (!pr && !historical) { - console.log("No CODEOWNERS-authorized profile-ci PR at this revision; no data exported."); - return; - } - const jobs = pages(`${repoPath}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`, "jobs"); - const artifacts = pages(`${repoPath}/actions/runs/${runId}/artifacts?per_page=100`, "artifacts") - .filter((artifact) => artifact.name.startsWith(`ci-profile-${attempt}-`)); - // The run name snapshots selection at dispatch. A label added while an - // ordinary run finishes must not retroactively select that earlier run. - if (!historical && !run.display_title.startsWith("CI profile · PR #")) { + // The run name snapshots label selection at dispatch. A label added while + // an ordinary run finishes must not retroactively select that earlier run; + // such a run is exported only if its run number is in the sample. + const mode = historical ? "historical" : exportMode(run); + if (!mode) { console.log("This attempt was not selected for profiling; no data exported."); return; } + const pr = mode === "label" ? associatedProfilePr(run, pulls, ownerIds) + : mode === "sampled" ? associatedOwnerPr(run, pulls, ownerIds) : undefined; + if (!pr && !historical) { + console.log("No CODEOWNERS-authorized PR at this revision; no data exported."); + return; + } + const jobs = pages(`${repoPath}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`, "jobs"); + // Sampled runs wrote no timing artifacts, so none are read. + const artifacts = mode === "sampled" ? [] : pages(`${repoPath}/actions/runs/${runId}/artifacts?per_page=100`, "artifacts") + .filter((artifact) => artifact.name.startsWith(`ci-profile-${attempt}-`)); + if (!/^[a-f\d]{40}$/.test(run.head_sha)) throw new Error("Invalid head commit"); + const loadChecks = () => normalizeChecks(run, { + checkRuns: pages(`${repoPath}/commits/${run.head_sha}/check-runs?per_page=100`, "check_runs"), + statuses: pages(`${repoPath}/commits/${run.head_sha}/statuses?per_page=100`), + workflowRuns: pages(`${repoPath}/actions/runs?head_sha=${run.head_sha}&per_page=100`, "workflow_runs"), + }); + const checks = await settledChecks(run, loadChecks, args.includes("--dry-run") ? { waitMs: 0 } : {}); const directory = mkdtempSync(path.join(tmpdir(), "paperclip-ci-export-")); const records = []; let bytes = 0; @@ -108,8 +134,11 @@ async function main() { } finally { rmSync(directory, { recursive: true, force: true }); } - const trace = buildTrace(run, jobs, records); - trace.events[0].data["ci.profile_source"] = historical ? "historical" : "label"; + const trace = buildTrace(run, jobs, records, { + sampleRate: mode === "sampled" ? sampleRate : 1, checks, testDetail: mode !== "sampled", + }); + trace.events[0].data["ci.profile_source"] = mode; + if (pr) trace.events[0].data["ci.pr_number"] = pr.number; const dataset = process.env.HONEYCOMB_CI_DATASET || "paperclip-ci"; const url = traceUrl(process.env.HONEYCOMB_TEAM || "paperclip", process.env.HONEYCOMB_ENVIRONMENT || "test", dataset, trace); if (arg("--output")) writeFileSync(arg("--output"), JSON.stringify(trace, null, 2)); @@ -123,6 +152,8 @@ async function main() { `### CI profile${pr ? ` for PR #${pr.number}` : " (historical job/step baseline)"}`, "", `[Open the full CI waterfall in Honeycomb](${url})`, "", `${trace.jobCount} jobs · ${(trace.wallMs / 1000).toFixed(1)} seconds elapsed · ${trace.events.length} spans`, + "", `${trace.checkCount} outside checks · ${(trace.windowMs / 1000).toFixed(1)} seconds until the last one finished`, + ...(mode === "sampled" ? ["", `Sampled run (1 in ${sampleRate}): job and step timings, without per-test detail.`] : []), "", `GitHub run: ${run.html_url}/attempts/${attempt}`, "", ...trace.warnings.map((warning) => `- ${warning}`), "", "The workflow root measures elapsed time. Parallel job durations must not be summed as elapsed time.", "", diff --git a/scripts/ci-profile/trace.mjs b/scripts/ci-profile/trace.mjs index a36112d6ced..ee149be2066 100644 --- a/scripts/ci-profile/trace.mjs +++ b/scripts/ci-profile/trace.mjs @@ -2,10 +2,73 @@ import { createHash } from "node:crypto"; const millis = (value) => Date.parse(value); const id = (value, size = 16) => createHash("sha256").update(value).digest("hex").slice(0, size); -const failed = (status) => ["failure", "failed", "timed_out", "timedOut", "cancelled", "interrupted"].includes(status); +// "error" is a commit status that could not complete. +const failed = (status) => ["failure", "failed", "error", "timed_out", "timedOut", "cancelled", "interrupted"].includes(status); const text = (value) => typeof value === "string" && value.length <= 4096; -export function buildTrace(run, jobs, records = []) { +// Outside checks belong to the push that started this run. A check that began +// well before the run (for example, before a later label event) is not part of +// this run's elapsed time. +const checkSlackMs = 60_000; +const label = (value) => typeof value === "string" ? value.slice(0, 256) : ""; + +// Reduce GitHub's check-run, commit-status and workflow-run listings for the +// run's head commit to the allowlisted fields the trace needs. +export function normalizeChecks(run, { checkRuns = [], statuses = [], workflowRuns = [] } = {}) { + const checks = []; + for (const check of checkRuns) { + // GitHub Actions check runs are jobs. This run's jobs are already spans, + // and other workflows are summarized per workflow below. + if (!check?.app?.slug || check.app.slug === "github-actions") continue; + checks.push({ + kind: "external_check", app: label(check.app.slug), name: label(check.name), + status: label(check.conclusion || check.status), startedAt: check.started_at, + completedAt: check.status === "completed" ? check.completed_at : null, + }); + } + const contexts = new Map(); + for (const status of statuses) { + if (typeof status?.context !== "string") continue; + contexts.set(status.context, [...(contexts.get(status.context) || []), status]); + } + for (const [context, entries] of contexts) { + entries.sort((a, b) => millis(a.created_at) - millis(b.created_at)); + const last = entries.at(-1); + checks.push({ + kind: "external_check", app: "commit-status", name: label(context), status: label(last.state), + startedAt: entries[0].created_at, completedAt: last.state === "pending" ? null : last.created_at, + }); + } + const workflows = new Map(); + for (const other of workflowRuns) { + if (other?.id === run.id || other?.path === run.path || other?.head_sha !== run.head_sha + || !["pull_request", "pull_request_target"].includes(other?.event)) continue; + const known = workflows.get(other.path); + if (!known || millis(other.created_at) > millis(known.created_at)) workflows.set(other.path, other); + } + for (const other of workflows.values()) { + checks.push({ + kind: "workflow_check", app: "github-actions", name: label(other.name), + status: label(other.conclusion || other.status), + startedAt: other.run_attempt > 1 ? other.run_started_at : other.created_at, + // GitHub exposes no completion time for a run; its last update is the closest record. + completedAt: other.status === "completed" ? other.updated_at : null, + }); + } + return checks; +} + +const runStart = (run) => millis(run.run_attempt === 1 ? run.created_at : run.run_started_at); +// A rerun repeats jobs, not the outside checks, so only a first attempt owns them. +const ownChecks = (run, checks) => run.run_attempt === 1 + ? checks.filter((check) => !(millis(check.startedAt) < runStart(run) - checkSlackMs)) : []; + +export function pendingChecks(run, checks, kind = "external_check") { + return ownChecks(run, checks).filter((check) => check.kind === kind && !check.completedAt); +} + +export function buildTrace(run, jobs, records = [], { sampleRate = 1, checks = [], testDetail = true } = {}) { + if (!Number.isSafeInteger(sampleRate) || sampleRate < 1) throw new Error("Invalid sample rate"); if (run.status !== "completed" || !jobs.length || jobs.some((job) => job.status !== "completed")) { throw new Error("A complete workflow attempt and its full job inventory are required"); } @@ -15,7 +78,7 @@ export function buildTrace(run, jobs, records = []) { const key = `${run.repository.full_name}:${run.id}:${run.run_attempt}`; const traceId = id(key, 32); const rootId = id(`${key}:root`); - const start = millis(run.run_attempt === 1 ? run.created_at : run.run_started_at); + const start = runStart(run); // The attempt jobs API can relabel carried-over successful jobs with the // new attempt number. Their completion clocks still precede this attempt. const carriedJobs = run.run_attempt > 1 ? jobs.filter((job) => millis(job.completed_at) < start) : []; @@ -39,7 +102,9 @@ export function buildTrace(run, jobs, records = []) { const span = (spanId, parent, name, from, to, fields) => { if (!Number.isFinite(from) || !Number.isFinite(to) || to < from) return; events.push({ - time: new Date(from).toISOString(), samplerate: 1, + // Sampling selects whole runs. Every span of a selected run is kept, and + // this weight lets Honeycomb scale counts back to all runs. + time: new Date(from).toISOString(), samplerate: sampleRate, data: { ...shared, "trace.span_id": spanId, ...(parent ? { "trace.parent_id": parent } : {}), name, duration_ms: to - from, ...fields }, }); @@ -87,6 +152,32 @@ export function buildTrace(run, jobs, records = []) { } } + // Reviews, scans and other workflows on the same commit run beside this + // workflow. They are siblings of the jobs on the same wall clock. + const own = ownChecks(run, checks); + const stale = run.run_attempt === 1 ? checks.length - own.length : 0; + const pending = own.filter((check) => !check.completedAt); + let lastEnd = end; + let lastName = "PR workflow"; + let slowest; + let exported = 0; + for (const check of own) { + const from = millis(check.startedAt); + const to = millis(check.completedAt); + // Skipped checks report a completion before their start; they did no work. + if (!check.completedAt || check.status === "skipped" || !Number.isFinite(from) || !Number.isFinite(to) || to < from) continue; + span(id(`${key}:check:${check.kind}:${check.app}:${check.name}`), rootId, check.name, from, to, { + "ci.kind": check.kind, "ci.check": check.name, "ci.check_app": check.app, + "ci.status": check.status, error: failed(check.status), + "ci.start_offset_ms": from - start, + "ci.finishes_after_workflow": to > end, + }); + exported++; + if (to > lastEnd) { lastEnd = to; lastName = check.name; } + if (!slowest || to - from > slowest.ms) slowest = { name: check.name, ms: to - from }; + } + if (pending.length) warnings.push(`${pending.length} outside checks were still running at export`); + const receipts = new Set(); const invocations = new Map(); const valid = []; @@ -138,7 +229,9 @@ export function buildTrace(run, jobs, records = []) { ...(Number.isFinite(record.setupMs) ? { "test.setup_ms": record.setupMs } : {}), }); } - const expectedReceipts = jobs.filter((job) => job.steps?.some((step) => step.name === "Initialize CI profile")); + // Ordinary runs skip the collection steps, so they owe no timing artifacts. + const expectedReceipts = testDetail + ? jobs.filter((job) => job.steps?.some((step) => step.name === "Initialize CI profile")) : []; const missing = expectedReceipts.filter((job) => !receipts.has(job.id)); if (missing.length) warnings.push(`Missing timing artifacts for ${missing.length} jobs`); // Some package scripts invoke a reporter without using the launcher. Its @@ -160,20 +253,33 @@ export function buildTrace(run, jobs, records = []) { return framework && !valid.some((entry) => entry.job.id === job.id && entry.record.framework === framework); }); if (expectedReceipts.length && missingTests.length) warnings.push(`Missing test detail for ${missingTests.length} test jobs (including jobs that failed before testing)`); + root["ci.sample_rate"] = sampleRate; + root["ci.workflow_ms"] = end - start; + // Push to the last finished check. Checks still running at export are excluded and counted. + root["ci.all_checks_ms"] = lastEnd - start; + root["ci.last_to_finish"] = lastName; + root["ci.check_count"] = exported; + root["ci.pending_check_count"] = pending.length; + root["ci.pending_checks"] = pending.map((check) => check.name).join(", ").slice(0, 1024); + root["ci.earlier_check_count"] = stale; + if (slowest) { + root["ci.slowest_check"] = slowest.name; + root["ci.slowest_check_ms"] = slowest.ms; + } root["ci.test_count"] = valid.filter(({ record }) => record.kind === "test").length; root["ci.file_count"] = valid.filter(({ record }) => record.kind === "file").length; root["ci.expected_invocations"] = invocations.size; root["ci.reported_invocations"] = [...invocations.values()].filter((value) => value.start && value.end).length; root["ci.profile_complete"] = warnings.length === 0 && run.run_attempt === 1; root["ci.coverage_warnings"] = warnings.join("; "); - return { traceId, events, warnings, wallMs: end - start, jobCount: jobs.length }; + return { traceId, events, warnings, wallMs: end - start, windowMs: lastEnd - start, jobCount: jobs.length, checkCount: exported }; } export function traceUrl(team, environment, dataset, trace) { const params = new URLSearchParams({ trace_id: trace.traceId, trace_start_ts: String(Math.floor(Date.parse(trace.events[0].time) / 1000)), - trace_end_ts: String(Math.ceil((Date.parse(trace.events[0].time) + trace.wallMs) / 1000)), + trace_end_ts: String(Math.ceil((Date.parse(trace.events[0].time) + (trace.windowMs ?? trace.wallMs)) / 1000)), }); return `https://ui.honeycomb.io/${encodeURIComponent(team)}/environments/${encodeURIComponent(environment)}/datasets/${encodeURIComponent(dataset)}/trace?${params}`; }