diff --git a/.github/scripts/tests/ci-plan.test.mjs b/.github/scripts/tests/ci-plan.test.mjs index 3285cc3b072..25e8580fb00 100644 --- a/.github/scripts/tests/ci-plan.test.mjs +++ b/.github/scripts/tests/ci-plan.test.mjs @@ -6,6 +6,7 @@ import path from "node:path"; import test from "node:test"; import { comparisonPaths, planTasks } from "../ci-plan.mjs"; import { mergeDiff } from "../ci-diff.mjs"; +import { planLane } from "../../../scripts/ci-typecheck-lane.mjs"; const root = new URL("../../../", import.meta.url); const config = JSON.parse(readFileSync(new URL("turbo.json", root), "utf8")); @@ -62,7 +63,9 @@ test("repository-wide guidance guards run uncached for every scanned UI file typ assert.equal(config.tasks["//#ci:typecheck"].cache, false); const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); const job = workflow.split("\n typecheck_release_registry:\n")[1].split("\n general_tests:\n")[0]; - assert.match(job, /- name: Verify repository-wide CLI guidance safety\n run: pnpm exec vitest run --project @paperclipai\/server server\/src\/__tests__\/cli-invocation-safety.test.ts/); + assert.match(job, /\n run: node scripts\/ci-typecheck-lane\.mjs\n/); + const guidance = planLane({ buildGapPackages: [], testLayers: [] }).tasks.find((task) => task.name === "CLI guidance safety"); + assert.deepEqual(guidance.args, ["exec", "vitest", "run", "--project", "@paperclipai/server", "server/src/__tests__/cli-invocation-safety.test.ts"]); assert.ok(!job.includes("uses: ./.github/actions/run-ci-task")); }); diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index e8a2df28761..391df4f854f 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -351,7 +351,7 @@ jobs: pnpm install --frozen-lockfile fi - # typecheck:build-gaps builds @paperclipai/server, whose build script + # The typecheck lane builds @paperclipai/server, whose build script # rebuilds the Runner release binary; without the shared Rust cache that # is a ~3m40s cold compile of all third-party crates (run 35036001734, # 2026-09-15). Same restore-only contract as Verify Paperclip Runner. @@ -462,25 +462,8 @@ jobs: if: ${{ steps.runner_release_crate.outputs.cache-hit == 'true' }} run: node scripts/runner-release-crate-cache.mjs mark-current - - name: Typecheck workspaces whose build scripts skip TypeScript - run: pnpm run typecheck:build-gaps - - - name: Typecheck test files that no build reads - run: node scripts/typecheck-test-programs.mjs --low-memory - - - name: Typecheck the server in low-memory layers - env: - PAPERCLIP_TYPECHECK_LOW_MEMORY: "1" - run: pnpm --filter @paperclipai/server typecheck - - - name: Typecheck test projects against the server declarations - run: node scripts/typecheck-test-projects.mjs --low-memory --reuse-declarations - - - name: Verify repository-wide CLI guidance safety - run: pnpm exec vitest run --project @paperclipai/server server/src/__tests__/cli-invocation-safety.test.ts - - - name: Verify release registry test coverage - run: pnpm run test:release-registry + - name: Typecheck, CLI guidance safety and release registry tests + run: node scripts/ci-typecheck-lane.mjs - name: Upload CI profile timings continue-on-error: true diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 6091d53f375..7a03bbebfdc 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1483,6 +1483,11 @@ The server typecheck builds only the Runner's TypeScript output. Use `pnpm --filter @paperclipai/paperclip-runner build` when a local check needs the Runner binary. +`node server/scripts/typecheck-layers.mjs --layer ` checks one layer +against the configs and declarations that an earlier `--declarations-only` run +left in that directory. It does not write them again. Stale declarations pass +silently, so use it only straight after `--declarations-only` on the same commit. + If a compiler child terminates on a signal, the wrapper reports the signal, layer, command, and working directory on stderr, then stops with a nonzero exit. Compiler stdout and stderr remain visible. A signal such as `SIGKILL` alone does @@ -1525,8 +1530,9 @@ available to the task is limited, add `--low-memory`, or set The command fails if a project loads server source, because that is what makes these programs large. It does not limit memory. CI runs it with -`--reuse-declarations` straight after the server typecheck, so it does not build -the declarations again. Do not use that option when the declarations can be stale. +`--reuse-declarations` after the server declarations of the same run, so it does +not build the declarations again. Do not use that option when the declarations +can be stale. Add each new test project that imports server source to `PROJECTS` in `scripts/typecheck-test-projects.mjs`. @@ -1538,6 +1544,43 @@ that file, and it fails when a referenced config does not exist yet. The program. That program peaks at about 15 GiB at default settings and about 8 GiB on one thread. +#### Typecheck lane in CI + +The required `Typecheck + Release Registry` job runs one command: + +```sh +node scripts/ci-typecheck-lane.mjs [--concurrency ] +``` + +It first runs the steps that write shared outputs, one at a time: + +1. Link the workspace packages and build the plugin SDK. +2. Build the server with `PAPERCLIP_TSC_EMIT_ONLY=1`, then check that the + server runtime assets are in `dist`. The server build no longer checks types + here: the server layers check the same files, and the `Build` job runs the + full `pnpm build`. The full program peaked at about 13.5 GiB, which left this + 16 GiB runner swapping. +3. Install the sandbox providers for the test programs + (`typecheck-test-programs.mjs --prepare-only`). +4. Typecheck the Runner. Its typecheck rebuilds the eval kernel, which other + checks read, so it cannot run beside them. + +Then it runs the remaining checks in parallel, as many as the machine has CPUs +(default) or `--concurrency`. Each `tsc` uses one thread. The server +declaration layers start first, because the test layers and the test projects +read them. The test programs, the other build-gap workspaces and the CLI +guidance test run at a lower CPU priority, so they use only the CPU that the +server layers and the release registry tests leave free. + +A passing check prints its output in a collapsed log group. A failing check +prints its output in full, between `=== FAILED ...` and `=== end of ...` +lines, with an error annotation. When the server declarations fail, the test +layers and test projects are reported as skipped. The lane ends with a table +of each check's start time, duration and result. If a preparation step fails, +nothing else runs. + +To add a check to the job, add it to `planLane` in the script. + #### Typecheck test files outside the build Most package tsconfigs exclude tests, and Vitest strips types, so a type error in @@ -1550,7 +1593,9 @@ node scripts/typecheck-test-programs.mjs --low-memory node scripts/typecheck-test-programs.mjs packages/paperclip-runner/tsconfig.test.json ``` -The first form runs in the required `Typecheck + Release Registry` job. It checks +The first form runs in the required `Typecheck + Release Registry` job, split in +two: `--prepare-only` installs the sandbox providers and builds what the programs +read, and `--skip-prepare` then only checks. It checks one program at a time and the largest peaks below 2 GiB. It then fails if a tracked `*.test.ts(x)` or `*.spec.ts(x)` file is in no program. The compiler's file list decides that, because `include` silently drops `foo.test.tsx` when `foo.test.ts` diff --git a/package.json b/package.json index e704ca28508..590fd8f5550 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/ci-typecheck-lane.test.mjs b/scripts/__tests__/ci-typecheck-lane.test.mjs new file mode 100644 index 00000000000..64107c85eca --- /dev/null +++ b/scripts/__tests__/ci-typecheck-lane.test.mjs @@ -0,0 +1,152 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { getPriority } from "node:os"; +import test from "node:test"; +import { BACKGROUND_PRIORITY, createReporter, planLane, runLane, SERIAL_BUILD_GAPS } from "../ci-typecheck-lane.mjs"; +import { PROJECTS } from "../typecheck-test-projects.mjs"; + +const plan = () => + planLane({ + buildGapPackages: ["paperclipai", "@paperclipai/paperclip-runner", "@paperclipai/ui"], + testLayers: ["tests", "test-tree-0"], + }); + +const quietReporter = (lines = []) => createReporter({ write: (text) => lines.push(text), github: false }); +const node = (name, source, needs) => ({ name, command: process.execPath, args: ["-e", source], needs }); + +test("the shared outputs are written before the parallel checks start", () => { + const { prepare, tasks } = plan(); + const names = prepare.map((task) => task.name); + const build = prepare.find((task) => task.args.includes("@paperclipai/server") && task.args.includes("build")); + assert.deepEqual(build.env, { PAPERCLIP_TSC_EMIT_ONLY: "1" }); + assert.ok(names.indexOf("check server runtime assets") > names.indexOf(build.name)); + assert.ok(prepare.some((task) => task.args.join(" ") === "scripts/typecheck-test-programs.mjs --prepare-only")); + for (const name of SERIAL_BUILD_GAPS) { + assert.ok(prepare.some((task) => task.name === `typecheck ${name}`), name); + assert.ok(!tasks.some((task) => task.name === `typecheck ${name}`), name); + } + const programs = tasks.find((task) => task.name === "typecheck test programs"); + assert.ok(programs.args.includes("--skip-prepare")); + for (const task of tasks) { + assert.ok(!(task.args.includes("@paperclipai/server") && task.args.includes("typecheck")), task.name); + } +}); + +test("every check of the sequential job is planned, and only declaration readers wait", () => { + const { tasks } = plan(); + const byName = new Map(tasks.map((task) => [task.name, task])); + assert.deepEqual(byName.get("server declarations").args, ["server/scripts/typecheck-layers.mjs", "--declarations-only"]); + for (const layer of ["tests", "test-tree-0"]) { + const task = byName.get(`server layer ${layer}`); + assert.deepEqual(task.args, ["server/scripts/typecheck-layers.mjs", "--layer", layer]); + assert.deepEqual(task.needs, ["server declarations"]); + } + for (const project of PROJECTS) { + const task = byName.get(`test project ${project}`); + assert.deepEqual(task.args.slice(-2), ["--reuse-declarations", project]); + assert.deepEqual(task.needs, ["server declarations"]); + } + for (const name of ["paperclipai", "@paperclipai/ui"]) assert.ok(byName.has(`typecheck ${name}`), name); + assert.deepEqual(byName.get("release registry tests").args, ["run", "test:release-registry"]); + assert.ok(byName.get("CLI guidance safety").args.includes("server/src/__tests__/cli-invocation-safety.test.ts")); + const waiting = tasks.filter((task) => task.needs); + assert.equal(waiting.length, 2 + PROJECTS.length); + assert.deepEqual( + tasks.filter((task) => !task.background).map((task) => task.name), + ["server declarations", "release registry tests", ...waiting.map((task) => task.name)], + ); +}); + +test("checks run at most the given number at a time, in priority order", async () => { + let running = 0; + let peak = 0; + const started = []; + const runTask = async (task) => { + started.push(task.name); + running += 1; + peak = Math.max(peak, running); + await new Promise((resolve) => setTimeout(resolve, 5)); + running -= 1; + return { status: 0, output: "", durationMs: 5 }; + }; + const tasks = ["a", "b", "c", "d", "e"].map((name) => ({ name, command: "x", args: [] })); + const status = await runLane({ prepare: [], tasks, concurrency: 2, runTask, reporter: quietReporter() }); + assert.equal(status, 0); + assert.equal(peak, 2); + assert.deepEqual(started, ["a", "b", "c", "d", "e"]); +}); + +test("a failed check fails the lane, skips its dependents and still runs the others", async () => { + const lines = []; + const status = await runLane({ + prepare: [], + tasks: [ + node("declarations", "console.log('src/a.ts(1,1): error TS2322'); process.exit(2)"), + node("dependent", "console.log('never')", ["declarations"]), + node("independent", "console.error('independent stderr')"), + ], + concurrency: 1, + reporter: quietReporter(lines), + }); + const output = lines.join(""); + assert.equal(status, 2); + assert.match(output, /FAILED in [\d.]+s with exit code 2: declarations[^\n]*\nsrc\/a\.ts\(1,1\): error TS2322\n=== end of declarations/); + assert.match(output, /skipped: dependent, because declarations did not pass/); + assert.doesNotMatch(output, /never/); + assert.match(output, /independent stderr/); + assert.match(output, /declarations\s+\S+\s+\S+\s+FAILED \(exit 2\)\n/); + assert.match(output, /dependent\s+skipped\n/); +}); + +test("a failed preparation step stops the lane before any check", async () => { + const ran = []; + const status = await runLane({ + prepare: [{ name: "prepare", command: "x", args: [] }], + tasks: [{ name: "check", command: "x", args: [] }], + concurrency: 4, + runPrepare: async () => ({ status: 7, durationMs: 1 }), + runTask: async (task) => { + ran.push(task.name); + return { status: 0, output: "", durationMs: 1 }; + }, + reporter: quietReporter(), + }); + assert.equal(status, 7); + assert.deepEqual(ran, []); +}); + +test("passing checks are folded into GitHub log groups and failures are annotated", () => { + const lines = []; + const reporter = createReporter({ write: (text) => lines.push(text), github: true }); + const task = { name: "check", command: "tsc", args: ["-p", "a.json"] }; + reporter.taskEnd(task, { status: 0, output: "ok", durationMs: 1200 }); + reporter.taskEnd(task, { status: 2, output: "a.ts: error\n", durationMs: 1200 }); + const output = lines.join(""); + assert.match(output, /^::group::passed in 1\.2s: check: tsc -p a\.json\nok\n::endgroup::\n/); + assert.match(output, /::error title=Typecheck lane::check failed with exit code 2\n\n=== FAILED in 1\.2s with exit code 2: check: tsc -p a\.json\na\.ts: error\n=== end of check\n/); +}); + +test("background checks run at a lower CPU priority than the critical path", { skip: process.platform === "win32" }, async () => { + const lines = []; + const report = "console.log('priority=' + require('node:os').getPriority())"; + await runLane({ + prepare: [], + tasks: [node("critical", report), { ...node("filler", report), background: true }], + concurrency: 2, + reporter: quietReporter(lines), + }); + const output = lines.join(""); + const own = getPriority(); + assert.match(output, new RegExp(`: critical: [^\\n]*\\npriority=${own}\\n`)); + assert.match(output, new RegExp(`: filler: [^\\n]*\\npriority=${Math.max(own, BACKGROUND_PRIORITY)}\\n`)); +}); + +test("the PR typecheck job runs the lane after the Runner release crate restore", () => { + const workflow = readFileSync(new URL("../../.github/workflows/pr-trusted.yml", import.meta.url), "utf8"); + const job = workflow.split("\n typecheck_release_registry:\n")[1].split("\n general_tests:\n")[0]; + const lane = job.indexOf("run: node scripts/ci-typecheck-lane.mjs"); + assert.ok(lane > job.indexOf("- name: Mark the restored Runner release crate current")); + for (const command of ["typecheck:build-gaps", "typecheck-test-programs.mjs", "typecheck-test-projects.mjs", "test:release-registry"]) { + assert.ok(!job.includes(command), command); + } +}); diff --git a/scripts/__tests__/runner-release-crate-cache.test.mjs b/scripts/__tests__/runner-release-crate-cache.test.mjs index 377eac9518c..72cabec8e16 100644 --- a/scripts/__tests__/runner-release-crate-cache.test.mjs +++ b/scripts/__tests__/runner-release-crate-cache.test.mjs @@ -46,7 +46,7 @@ const RUST_RESTORE_STEP = "Restore Runner Rust dependencies (read only)"; // Every PR job that builds the Runner release binary, and the command each // one runs cargo through. The crate must be restored and marked before it. const READERS = [ - ["typecheck_release_registry", "run: pnpm run typecheck:build-gaps"], + ["typecheck_release_registry", "run: node scripts/ci-typecheck-lane.mjs"], ["verify_paperclip_runner", "run: pnpm --filter @paperclipai/paperclip-runner ${{ matrix.command }}"], ["build", "run: pnpm build"], ["canary_dry_run", "- name: Release canary dry run via release.sh internal build"], diff --git a/scripts/ci-typecheck-lane.mjs b/scripts/ci-typecheck-lane.mjs new file mode 100644 index 00000000000..312e9c57207 --- /dev/null +++ b/scripts/ci-typecheck-lane.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node + +import { spawn, spawnSync } from "node:child_process"; +import { availableParallelism, setPriority } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { listLayers } from "../server/scripts/typecheck-layers.mjs"; +import { LOW_MEMORY_VARIABLE, EMIT_ONLY_VARIABLE } from "./run-tsc.mjs"; +import { PROJECTS } from "./typecheck-test-projects.mjs"; + +export const SERIAL_BUILD_GAPS = ["@paperclipai/paperclip-runner"]; +export const BACKGROUND_PRIORITY = 10; +const SERVER_DECLARATIONS = "server declarations"; +const lowMemory = { [LOW_MEMORY_VARIABLE]: "1" }; + +export function planLane({ buildGapPackages, testLayers, projects = PROJECTS }) { + const typecheckPackage = (name, background) => ({ + name: `typecheck ${name}`, + command: "pnpm", + args: ["--filter", name, "typecheck"], + env: lowMemory, + background, + }); + const prepare = [ + { name: "link workspace packages", command: "pnpm", args: ["run", "preflight:workspace-links"] }, + { name: "build plugin SDK", command: "pnpm", args: ["--filter", "@paperclipai/plugin-sdk", "ensure-build-deps"] }, + { + name: "build server without type checks", + command: "pnpm", + args: ["--filter", "@paperclipai/server", "build"], + env: { [EMIT_ONLY_VARIABLE]: "1" }, + }, + { + name: "check server runtime assets", + command: "node", + args: ["scripts/run-typecheck-build-gaps.mjs", "--runtime-assets-only"], + }, + { name: "install test program dependencies", command: "node", args: ["scripts/typecheck-test-programs.mjs", "--prepare-only"] }, + ...buildGapPackages.filter((name) => SERIAL_BUILD_GAPS.includes(name)).map((name) => typecheckPackage(name, false)), + ]; + const needsDeclarations = [SERVER_DECLARATIONS]; + const tasks = [ + { + name: SERVER_DECLARATIONS, + command: "node", + args: ["server/scripts/typecheck-layers.mjs", "--declarations-only"], + env: lowMemory, + }, + { name: "release registry tests", command: "pnpm", args: ["run", "test:release-registry"] }, + { + name: "typecheck test programs", + command: "node", + args: ["scripts/typecheck-test-programs.mjs", "--low-memory", "--skip-prepare"], + background: true, + }, + ...testLayers.map((layer) => ({ + name: `server layer ${layer}`, + command: "node", + args: ["server/scripts/typecheck-layers.mjs", "--layer", layer], + env: lowMemory, + needs: needsDeclarations, + })), + ...projects.map((project) => ({ + name: `test project ${project}`, + command: "node", + args: ["scripts/typecheck-test-projects.mjs", "--low-memory", "--reuse-declarations", project], + needs: needsDeclarations, + })), + ...buildGapPackages.filter((name) => !SERIAL_BUILD_GAPS.includes(name)).map((name) => typecheckPackage(name, true)), + { + name: "CLI guidance safety", + command: "pnpm", + args: ["exec", "vitest", "run", "--project", "@paperclipai/server", "server/src/__tests__/cli-invocation-safety.test.ts"], + background: true, + }, + ]; + return { prepare, tasks }; +} + +const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`; + +function spawnTask(task, { cwd, env, stdio }) { + return spawn(task.command, task.args, { + cwd, + env: { ...env, ...task.env }, + stdio, + shell: process.platform === "win32", + }); +} + +function runCaptured(task, options) { + return new Promise((resolve) => { + const started = Date.now(); + const output = []; + let child; + try { + child = spawnTask(task, { ...options, stdio: ["ignore", "pipe", "pipe"] }); + } catch (error) { + resolve({ status: 1, output: `${error.message}\n`, durationMs: Date.now() - started }); + return; + } + if (task.background && child.pid) { + try { + setPriority(child.pid, BACKGROUND_PRIORITY); + } catch {} + } + child.stdout.on("data", (chunk) => output.push(chunk)); + child.stderr.on("data", (chunk) => output.push(chunk)); + child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`))); + child.on("close", (code, signal) => { + if (signal) output.push(Buffer.from(`${task.name}: terminated by ${signal}\n`)); + resolve({ status: signal ? 1 : (code ?? 1), output: Buffer.concat(output).toString("utf8"), durationMs: Date.now() - started }); + }); + }); +} + +function runStreamed(task, options) { + return new Promise((resolve) => { + const started = Date.now(); + let child; + try { + child = spawnTask(task, { ...options, stdio: "inherit" }); + } catch (error) { + resolve({ status: 1, output: `${error.message}\n`, durationMs: Date.now() - started }); + return; + } + child.on("error", (error) => process.stderr.write(`${error.message}\n`)); + child.on("close", (code, signal) => { + if (signal) process.stderr.write(`${task.name}: terminated by ${signal}\n`); + resolve({ status: signal ? 1 : (code ?? 1), durationMs: Date.now() - started }); + }); + }); +} + +export function createReporter({ write = (text) => process.stdout.write(text), github = Boolean(process.env.GITHUB_ACTIONS) } = {}) { + const groupStart = (title) => write(github ? `::group::${title}\n` : `\n=== ${title}\n`); + const groupEnd = () => write(github ? "::endgroup::\n" : ""); + const ensureNewline = (text) => (text.length === 0 || text.endsWith("\n") ? text : `${text}\n`); + return { + prepareStart(task) { + groupStart(`${task.name}: ${task.command} ${task.args.join(" ")}`); + }, + prepareEnd(task, result) { + groupEnd(); + write(`${result.status === 0 ? "passed" : "FAILED"}: ${task.name} (${seconds(result.durationMs)})\n`); + }, + taskEnd(task, result) { + const title = `${task.name}: ${task.command} ${task.args.join(" ")}`; + const output = ensureNewline(result.output); + if (result.status === 0) { + groupStart(`passed in ${seconds(result.durationMs)}: ${title}`); + write(output); + groupEnd(); + return; + } + if (github) write(`::error title=Typecheck lane::${task.name} failed with exit code ${result.status}\n`); + write(`\n=== FAILED in ${seconds(result.durationMs)} with exit code ${result.status}: ${title}\n${output}=== end of ${task.name}\n`); + }, + taskSkipped(task, missing) { + write(`skipped: ${task.name}, because ${missing.join(", ")} did not pass\n`); + }, + summary(rows) { + const width = Math.max(...rows.map((row) => row.name.length)); + write("\nTypecheck lane summary (start offset, duration, result):\n"); + for (const row of rows) { + const start = row.startMs === undefined ? "" : seconds(row.startMs); + const duration = row.durationMs === undefined ? "" : seconds(row.durationMs); + write(` ${row.name.padEnd(width)} ${start.padStart(7)} ${duration.padStart(7)} ${row.result}\n`); + } + }, + }; +} + +export async function runLane({ + prepare, + tasks, + concurrency, + cwd = process.cwd(), + env = process.env, + runPrepare = runStreamed, + runTask = runCaptured, + reporter = createReporter(), + now = Date.now, +}) { + const laneStart = now(); + const rows = []; + for (const task of prepare) { + reporter.prepareStart(task); + const startMs = now() - laneStart; + const result = await runPrepare(task, { cwd, env }); + reporter.prepareEnd(task, result); + rows.push({ name: task.name, startMs, durationMs: result.durationMs, result: result.status === 0 ? "passed" : "FAILED" }); + if (result.status !== 0) { + for (const remaining of [...prepare.slice(prepare.indexOf(task) + 1), ...tasks]) { + rows.push({ name: remaining.name, result: "not run" }); + } + reporter.summary(rows); + return result.status || 1; + } + } + + const states = new Map(tasks.map((task) => [task.name, "pending"])); + const results = new Map(); + const running = new Set(); + await new Promise((resolve) => { + const schedule = () => { + for (const task of tasks) { + if (states.get(task.name) !== "pending") continue; + const needs = task.needs ?? []; + const missing = needs.filter((name) => ["failed", "skipped"].includes(states.get(name))); + if (missing.length > 0) { + states.set(task.name, "skipped"); + reporter.taskSkipped(task, missing); + continue; + } + if (running.size >= concurrency || !needs.every((name) => states.get(name) === "passed")) continue; + states.set(task.name, "running"); + const startMs = now() - laneStart; + const promise = runTask(task, { cwd, env }).then((result) => { + running.delete(promise); + states.set(task.name, result.status === 0 ? "passed" : "failed"); + results.set(task.name, { ...result, startMs }); + reporter.taskEnd(task, result); + schedule(); + }); + running.add(promise); + } + if (running.size === 0) resolve(); + }; + schedule(); + }); + + let exitCode = 0; + for (const task of tasks) { + const state = states.get(task.name); + const result = results.get(task.name); + if (state !== "passed") exitCode ||= result?.status || 1; + rows.push({ + name: task.name, + startMs: result?.startMs, + durationMs: result?.durationMs, + result: state === "passed" ? "passed" : state === "failed" ? `FAILED (exit ${result.status})` : "skipped", + }); + } + reporter.summary(rows); + return exitCode; +} + +function listBuildGapPackages(root) { + const result = spawnSync(process.execPath, ["scripts/run-typecheck-build-gaps.mjs", "--list-packages"], { + cwd: root, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }); + if (result.error) throw result.error; + if (result.status !== 0) return null; + return JSON.parse(result.stdout.trim().split("\n").at(-1)); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = process.argv.slice(2); + const concurrencyIndex = args.indexOf("--concurrency"); + const concurrency = concurrencyIndex === -1 ? availableParallelism() : Number(args[concurrencyIndex + 1]); + const extra = args.filter((_, index) => concurrencyIndex === -1 || (index !== concurrencyIndex && index !== concurrencyIndex + 1)); + if (!Number.isInteger(concurrency) || concurrency < 1 || extra.length > 0) { + console.error("Usage: node scripts/ci-typecheck-lane.mjs [--concurrency ]"); + process.exitCode = 2; + } else { + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const buildGapPackages = listBuildGapPackages(root); + if (buildGapPackages === null) { + process.exitCode = 1; + } else { + const testLayers = listLayers(path.join(root, "server")) + .filter((layer) => !layer.emitsDeclarations) + .map((layer) => layer.name); + console.log(`typecheck lane: ${concurrency} at a time; build gaps: ${buildGapPackages.join(", ")}`); + process.exitCode = await runLane({ ...planLane({ buildGapPackages, testLayers }), concurrency, cwd: root }); + } + } +} diff --git a/scripts/run-typecheck-build-gaps.mjs b/scripts/run-typecheck-build-gaps.mjs index 9c2320969e8..964813c4bd1 100644 --- a/scripts/run-typecheck-build-gaps.mjs +++ b/scripts/run-typecheck-build-gaps.mjs @@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process"; const repoRoot = process.cwd(); const args = new Set(process.argv.slice(2)); -const allowedArgs = new Set(["--runtime-assets-only"]); +const allowedArgs = new Set(["--runtime-assets-only", "--list-packages"]); for (const arg of args) { if (!allowedArgs.has(arg)) { @@ -171,6 +171,11 @@ const buildGapPackages = buildGapCandidates.filter( ({ pkg }) => typeof pkg.scripts?.typecheck === "string", ); +if (args.has("--list-packages")) { + console.log(JSON.stringify(buildGapPackages.map(({ name }) => name))); + process.exit(0); +} + console.log( `[typecheck:build-gaps] typechecking ${buildGapPackages.length} workspace(s): ${buildGapPackages.map(({ name }) => name).join(", ") || "(none)"}`, ); diff --git a/scripts/typecheck-test-programs.mjs b/scripts/typecheck-test-programs.mjs index 3d5dcc26e2d..0cb114d274c 100644 --- a/scripts/typecheck-test-programs.mjs +++ b/scripts/typecheck-test-programs.mjs @@ -159,15 +159,11 @@ export function checkCoverage({ return findUncheckedTests({ trackedFiles: trackedFiles(root), checkedFiles }); } -export function typecheckTestPrograms({ +export function prepareTestPrograms({ root, env = process.env, - programs = PROGRAMS, - coverage = true, pnpm = runPnpm, linkPlugins = linkExcludedPlugins, - run = runTsc, - checkFiles = checkCoverage, log = console.log, logError = console.error, }) { @@ -197,6 +193,26 @@ export function typecheckTestPrograms({ } // The installs remove the SDK link from each provider's node_modules. linkPlugins(); + return 0; +} + +export function typecheckTestPrograms({ + root, + env = process.env, + programs = PROGRAMS, + coverage = true, + prepare = true, + pnpm = runPnpm, + linkPlugins = linkExcludedPlugins, + run = runTsc, + checkFiles = checkCoverage, + log = console.log, + logError = console.error, +}) { + if (prepare) { + const prepared = prepareTestPrograms({ root, env, pnpm, linkPlugins, log, logError }); + if (prepared !== 0) return prepared; + } let exitCode = 0; for (const config of programs) { @@ -224,20 +240,28 @@ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.me const args = process.argv.slice(2); const flags = args.filter((arg) => arg.startsWith("--")); const programs = args.filter((arg) => !arg.startsWith("--")); - const unknownFlags = flags.filter((flag) => flag !== "--low-memory"); + const unknownFlags = flags.filter((flag) => !["--low-memory", "--prepare-only", "--skip-prepare"].includes(flag)); const unknown = programs.filter((program) => !PROGRAMS.includes(program)); + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const env = flags.includes("--low-memory") ? { ...process.env, [LOW_MEMORY_VARIABLE]: "1" } : process.env; if (unknownFlags.length > 0) { console.error(`Unknown option: ${unknownFlags.join(", ")}`); process.exitCode = 2; + } else if (flags.includes("--prepare-only") && (flags.includes("--skip-prepare") || programs.length > 0)) { + console.error("--prepare-only takes no programs and cannot be combined with --skip-prepare"); + process.exitCode = 2; } else if (unknown.length > 0) { console.error(`Unknown program: ${unknown.join(", ")}. Known programs:\n ${PROGRAMS.join("\n ")}`); process.exitCode = 2; + } else if (flags.includes("--prepare-only")) { + process.exitCode = prepareTestPrograms({ root, env }); } else { process.exitCode = typecheckTestPrograms({ - root: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), - env: flags.includes("--low-memory") ? { ...process.env, [LOW_MEMORY_VARIABLE]: "1" } : process.env, + root, + env, programs: programs.length > 0 ? programs : PROGRAMS, coverage: programs.length === 0, + prepare: !flags.includes("--skip-prepare"), }); } } diff --git a/scripts/typecheck-test-programs.test.mjs b/scripts/typecheck-test-programs.test.mjs index 3fb020597fa..179892e81fa 100644 --- a/scripts/typecheck-test-programs.test.mjs +++ b/scripts/typecheck-test-programs.test.mjs @@ -10,6 +10,7 @@ import { findUncheckedTests, LOCKED_PROVIDERS, PROGRAMS, + prepareTestPrograms, PROVIDER_WORKSPACE, RESOLVE_ARGS, SANDBOX_PROVIDERS, @@ -122,6 +123,20 @@ test("providers install frozen without scripts, then relink the SDK", () => { assert.equal(calls.linked, 1); }); +test("preparation runs on its own, and a run that skips it only checks", () => { + const prepared = harness(); + assert.equal(prepareTestPrograms(prepared.options), 0); + assert.ok(prepared.calls.pnpm.some(({ cwd }) => cwd === path.join("/repo", PROVIDER_WORKSPACE))); + assert.equal(prepared.calls.linked, 1); + assert.deepEqual(prepared.calls.tsc, []); + + const checked = harness(); + assert.equal(typecheckTestPrograms({ ...checked.options, prepare: false }), 0); + assert.deepEqual(checked.calls.pnpm, []); + assert.equal(checked.calls.linked, 0); + assert.equal(checked.calls.tsc.length, 2); +}); + test("a locked provider installs frozen from its own lockfile, outside any workspace", () => { const { options, calls } = harness(); assert.equal(typecheckTestPrograms(options), 0); diff --git a/server/scripts/typecheck-layers.mjs b/server/scripts/typecheck-layers.mjs index ea775d40e67..840b1a5c142 100644 --- a/server/scripts/typecheck-layers.mjs +++ b/server/scripts/typecheck-layers.mjs @@ -27,7 +27,7 @@ // // Usage: node scripts/typecheck-layers.mjs [--declarations-only] -import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { crc32 } from "node:zlib"; @@ -251,16 +251,40 @@ export function declarationsConfig(layers) { }; } +function planServerLayers(serverRoot) { + const sourceFiles = listSourceFiles(serverRoot); + return planLayers(new Map(sourceFiles.map((file) => [file, readFileSync(path.join(serverRoot, file), "utf8")]))); +} + +export function listLayers(serverRoot) { + return planServerLayers(serverRoot).map((layer) => ({ name: layer.name, emitsDeclarations: emitsDeclarations(layer) })); +} + +function checkOneLayer({ serverRoot, name, run, log, logError }) { + const config = path.join(OUTPUT_DIRECTORY, `tsconfig.${name}.json`); + if (!existsSync(path.join(serverRoot, config))) { + logError(`typecheck:layers: ${config} does not exist. Run typecheck-layers.mjs --declarations-only first.`); + return 2; + } + log(`typecheck:layers: ${name}`); + const { status, signal } = run(["-p", config], { cwd: serverRoot }); + if (signal) { + logError(`typecheck:layers: ${name}: compiler terminated by ${signal} (tsc -p ${config}, cwd: ${serverRoot})`); + return status || 1; + } + return status; +} + export function typecheckLayers({ serverRoot, declarationsOnly = false, + layer: only, run = runTsc, log = console.log, + logError = console.error, }) { - const sourceFiles = listSourceFiles(serverRoot); - const layers = planLayers( - new Map(sourceFiles.map((file) => [file, readFileSync(path.join(serverRoot, file), "utf8")])), - ); + if (only) return checkOneLayer({ serverRoot, name: only, run, log, logError }); + const layers = planServerLayers(serverRoot); const planned = new Set(layers.map((layer) => layer.name)); const emptyLayers = new Set(["below", "cycle", "above", "tests"].filter((name) => !planned.has(name))); @@ -289,9 +313,7 @@ export function typecheckLayers({ const { status, signal } = run(["-p", config], { cwd: serverRoot }); // A killed process leaves no declarations, so later layers would only add noise. if (signal) { - console.error( - `typecheck:layers: ${layer.name}: compiler terminated by ${signal} (tsc -p ${config}, cwd: ${serverRoot})`, - ); + logError(`typecheck:layers: ${layer.name}: compiler terminated by ${signal} (tsc -p ${config}, cwd: ${serverRoot})`); return status || 1; } // Later layers still run after type errors. Each file is checked in one @@ -302,8 +324,21 @@ export function typecheckLayers({ } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - process.exitCode = typecheckLayers({ - serverRoot: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), - declarationsOnly: process.argv.includes("--declarations-only"), - }); + const args = process.argv.slice(2); + const layerIndex = args.indexOf("--layer"); + const layer = layerIndex === -1 ? undefined : args[layerIndex + 1]; + const rest = args.filter((_, index) => layerIndex === -1 || (index !== layerIndex && index !== layerIndex + 1)); + const valid = + rest.every((arg) => arg === "--declarations-only") && + (layerIndex === -1 || (Boolean(layer) && !layer.startsWith("--") && rest.length === 0)); + if (!valid) { + console.error("Usage: node scripts/typecheck-layers.mjs [--declarations-only | --layer ]"); + process.exitCode = 2; + } else { + process.exitCode = typecheckLayers({ + serverRoot: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + declarationsOnly: args.includes("--declarations-only"), + layer, + }); + } } diff --git a/server/scripts/typecheck-layers.test.mjs b/server/scripts/typecheck-layers.test.mjs index 8239fea0c4d..123d5e6a928 100644 --- a/server/scripts/typecheck-layers.test.mjs +++ b/server/scripts/typecheck-layers.test.mjs @@ -9,7 +9,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { emitOnlyTscArgs, lowMemoryTscArgs, runTsc } from "../../scripts/run-tsc.mjs"; import { STEPS, typecheckLowMemory } from "../../scripts/typecheck-low-memory.mjs"; import { findServerSource, PREPARE_STEPS, PROJECTS, typecheckTestProjects } from "../../scripts/typecheck-test-projects.mjs"; -import { declarationsConfig, layerConfig, listSourceFiles, planLayers, readImports, typecheckLayers } from "./typecheck-layers.mjs"; +import { declarationsConfig, layerConfig, listLayers, listSourceFiles, planLayers, readImports, typecheckLayers } from "./typecheck-layers.mjs"; const serverRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const sourcesOf = (layers, name) => layers.find((layer) => layer.name === name)?.sourceFiles; @@ -286,6 +286,38 @@ describe("typecheckLayers", () => { ]); }); + it("checks one layer against the configs a declarations run left, without writing them again", () => { + const root = serverFixture(); + const silent = { log: () => {}, logError: () => {}, run: () => ({ status: 0, signal: null }) }; + expect(typecheckLayers({ serverRoot: root, layer: "tests", ...silent })).toBe(2); + + expect(typecheckLayers({ serverRoot: root, declarationsOnly: true, ...silent })).toBe(0); + const output = path.join(root, "node_modules", ".cache", "paperclip-typecheck"); + const marker = path.join(output, "dts", "marker.d.ts"); + mkdirSync(path.dirname(marker), { recursive: true }); + writeFileSync(marker, ""); + const configs = []; + const status = typecheckLayers({ + serverRoot: root, + layer: "tests", + log: () => {}, + run: (args, { cwd }) => { + configs.push({ config: args[1], cwd }); + return { status: 3, signal: null }; + }, + }); + + expect(status).toBe(3); + expect(configs).toEqual([{ config: path.join("node_modules", ".cache", "paperclip-typecheck", "tsconfig.tests.json"), cwd: root }]); + expect(existsSync(marker)).toBe(true); + expect(listLayers(root)).toEqual([ + { name: "below", emitsDeclarations: true }, + { name: "cycle", emitsDeclarations: true }, + { name: "above", emitsDeclarations: true }, + { name: "tests", emitsDeclarations: false }, + ]); + }); + it("reports type errors from every layer", () => { const root = serverFixture(); const configs = []; diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index b5c65ee2abf..544b074a427 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "e704ca28508fa7c887f704fc26603cf8fd623867", + "package.json": "590fd8f555086a3bdc4dfa41a8016a89be13b977", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284",