diff --git a/.github/actions/run-ci-task/action.yml b/.github/actions/run-ci-task/action.yml new file mode 100644 index 00000000000..5174ede4efe --- /dev/null +++ b/.github/actions/run-ci-task/action.yml @@ -0,0 +1,38 @@ +name: Run incremental CI task +description: Restore PR-scoped Turbo results and run a task with its complete input hash. +inputs: + task: + required: true + description: Root task from turbo.json. + args: + default: '[]' + description: JSON array of command arguments. + partition: + default: '0' + description: Matrix partition for independent cache uploads. + force: + default: 'false' + description: Execute tasks even when cached, including prerequisite builds. +runs: + using: composite + steps: + - name: Restore task results + if: inputs.force != 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .turbo/cache + key: turbo-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-${{ inputs.partition }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: turbo-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-${{ inputs.partition }}- + - name: Run task + shell: bash + env: + CI_TASK: ${{ inputs.task }} + CI_TASK_ARGS: ${{ inputs.args }} + CI_FORCE_TASKS: ${{ inputs.force }} + run: node scripts/ci/run-task.mjs + - name: Save successful task results + if: success() && inputs.force != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .turbo/cache + key: turbo-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-${{ inputs.partition }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/ci-task-matrix.json b/.github/ci-task-matrix.json new file mode 100644 index 00000000000..b0caf7a5412 --- /dev/null +++ b/.github/ci-task-matrix.json @@ -0,0 +1,283 @@ +{ + "typecheck_release_registry": [ + { + "task": "ci:typecheck" + } + ], + "general_tests": [ + { + "group": "general-server-without-chat", + "group_label": "server (1/12)", + "shard_index": 0, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (2/12)", + "shard_index": 1, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (3/12)", + "shard_index": 2, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (4/12)", + "shard_index": 3, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (5/12)", + "shard_index": 4, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (6/12)", + "shard_index": 5, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (7/12)", + "shard_index": 6, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (8/12)", + "shard_index": 7, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (9/12)", + "shard_index": 8, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (10/12)", + "shard_index": 9, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (11/12)", + "shard_index": 10, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-server-without-chat", + "group_label": "server (12/12)", + "shard_index": 11, + "shard_count": 12, + "task": "ci:server" + }, + { + "group": "general-chat", + "group_label": "chat (1/3)", + "shard_index": 0, + "shard_count": 3, + "task": "ci:chat" + }, + { + "group": "general-chat", + "group_label": "chat (2/3)", + "shard_index": 1, + "shard_count": 3, + "task": "ci:chat" + }, + { + "group": "general-chat", + "group_label": "chat (3/3)", + "shard_index": 2, + "shard_count": 3, + "task": "ci:chat" + }, + { + "group": "general-workspaces-a", + "group_label": "workspaces-a (1/2)", + "shard_index": 0, + "shard_count": 2, + "task": "ci:workspaces-a" + }, + { + "group": "general-workspaces-a", + "group_label": "workspaces-a (2/2)", + "shard_index": 1, + "shard_count": 2, + "task": "ci:workspaces-a" + }, + { + "group": "general-workspaces-b", + "group_label": "workspaces-b", + "task": "ci:workspaces-b" + }, + { + "group": "general-workspaces-c", + "group_label": "workspaces-c", + "task": "ci:db" + } + ], + "docker_context_integrity": [ + { + "task": "ci:docker" + } + ], + "verify_paperclip_runner": [ + { + "lane_label": "static checks", + "command": "check:static", + "task": "ci:runner-static" + }, + { + "lane_label": "rust", + "command": "check:runner", + "release_crate": true, + "task": "ci:runner-rust" + }, + { + "lane_label": "vitest 1/2", + "command": "test:typescript:vitest --shard=1/2", + "task": "ci:runner-vitest" + }, + { + "lane_label": "vitest 2/2", + "command": "test:typescript:vitest --shard=2/2", + "release_crate": true, + "task": "ci:runner-vitest" + } + ], + "build": [ + { + "task": "ci:build" + } + ], + "verify_serialized_server": [ + { + "shard_index": 0, + "shard_count": 9, + "shard_label": "1/9", + "task": "ci:serialized" + }, + { + "shard_index": 1, + "shard_count": 9, + "shard_label": "2/9", + "task": "ci:serialized" + }, + { + "shard_index": 2, + "shard_count": 9, + "shard_label": "3/9", + "task": "ci:serialized" + }, + { + "shard_index": 3, + "shard_count": 9, + "shard_label": "4/9", + "task": "ci:serialized" + }, + { + "shard_index": 4, + "shard_count": 9, + "shard_label": "5/9", + "task": "ci:serialized" + }, + { + "shard_index": 5, + "shard_count": 9, + "shard_label": "6/9", + "task": "ci:serialized" + }, + { + "shard_index": 6, + "shard_count": 9, + "shard_label": "7/9", + "task": "ci:serialized" + }, + { + "shard_index": 7, + "shard_count": 9, + "shard_label": "8/9", + "task": "ci:serialized" + }, + { + "shard_index": 8, + "shard_count": 9, + "shard_label": "9/9", + "task": "ci:serialized" + } + ], + "canary_dry_run": [ + { + "task": "ci:canary" + } + ], + "e2e_shards": [ + { + "shard_index": 0, + "shard_count": 8, + "shard_label": "1/8", + "task": "ci:e2e" + }, + { + "shard_index": 1, + "shard_count": 8, + "shard_label": "2/8", + "task": "ci:e2e" + }, + { + "shard_index": 2, + "shard_count": 8, + "shard_label": "3/8", + "task": "ci:e2e" + }, + { + "shard_index": 3, + "shard_count": 8, + "shard_label": "4/8", + "task": "ci:e2e" + }, + { + "shard_index": 4, + "shard_count": 8, + "shard_label": "5/8", + "task": "ci:e2e" + }, + { + "shard_index": 5, + "shard_count": 8, + "shard_label": "6/8", + "task": "ci:e2e" + }, + { + "shard_index": 6, + "shard_count": 8, + "shard_label": "7/8", + "task": "ci:e2e" + }, + { + "shard_index": 7, + "shard_count": 8, + "shard_label": "8/8", + "task": "ci:e2e" + } + ] +} diff --git a/.github/scripts/ci-plan.mjs b/.github/scripts/ci-plan.mjs new file mode 100644 index 00000000000..3019b936ecf --- /dev/null +++ b/.github/scripts/ci-plan.mjs @@ -0,0 +1,106 @@ +import { appendFileSync, readFileSync, realpathSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { matchesGlob } from "node:path"; +import { fileURLToPath } from "node:url"; + +const releaseNote = /^releases\/(beta\/)?v[^/]+\.md$/; +const validPath = (file) => typeof file === "string" && file.length > 0 + && !file.startsWith("/") && !file.split("/").includes("..") && !file.includes("\0"); + +function validateGlobs(globs) { + if (!Array.isArray(globs) || globs.some((glob) => !validPath(glob) || /[!{}[\]?$\\]/.test(glob))) { + throw new Error("Unsupported task input pattern"); + } +} + +export function comparisonPaths(comparison) { + if (!Array.isArray(comparison?.files) || comparison.files.length === 0 || comparison.files.length >= 300) { + throw new Error("Missing, empty, or possibly truncated comparison"); + } + return comparison.files.flatMap((file) => { + if (!validPath(file.filename) || (file.status === "renamed" && !validPath(file.previous_filename)) + || (file.previous_filename !== undefined && !validPath(file.previous_filename))) { + throw new Error("Malformed changed file"); + } + return file.previous_filename ? [file.previous_filename, file.filename] : [file.filename]; + }); +} + +export function planTasks({ config, lanes, paths, mode = "enabled", forceReason = null }) { + if (!["enabled", "shadow", "full"].includes(mode)) throw new Error("Invalid CI mode"); + validateGlobs(config.globalDependencies); + const tasks = config.tasks; + const inputs = new Map(); + function taskInputs(name, parents = []) { + if (inputs.has(name)) return inputs.get(name); + if (parents.includes(name)) throw new Error("Cyclic task graph"); + const task = tasks[`//#${name}`]; + if (!task) throw new Error(`Missing task: ${name}`); + validateGlobs(task.inputs); + const all = [...task.inputs]; + for (const dependency of task.dependsOn ?? []) { + if (!dependency.startsWith("//#")) throw new Error("Unsupported task dependency"); + all.push(...taskInputs(dependency.slice(3), [...parents, name])); + } + inputs.set(name, [...new Set(all)]); + return inputs.get(name); + } + for (const entries of Object.values(lanes)) { + if (!Array.isArray(entries) || !entries.length) throw new Error("Empty CI lane"); + for (const entry of entries) taskInputs(entry.task); + } + const changed = [...new Set(paths)]; + if (!changed.length || changed.some((file) => !validPath(file))) forceReason ??= "Missing or invalid diff"; + const matches = (file, globs) => globs.some((glob) => matchesGlob(file, glob)); + const globalChanges = changed.filter((file) => matches(file, config.globalDependencies)); + if (globalChanges.length) forceReason ??= `Global CI input changed: ${globalChanges[0]}`; + const unknown = changed.filter((file) => !releaseNote.test(file) + && !matches(file, config.globalDependencies) && ![...inputs.values()].some((globs) => matches(file, globs))); + if (unknown.length) forceReason ??= `Unclassified path: ${unknown[0]}`; + if (mode === "full") forceReason ??= "Full CI requested"; + const reasons = Object.fromEntries([...inputs].map(([task, globs]) => [task, + forceReason ? [forceReason] : [...new Set([...globalChanges, ...changed.filter((file) => matches(file, globs))])], + ])); + const jobs = Object.fromEntries(Object.entries(lanes).map(([job, entries]) => { + const affected = entries.filter((entry) => reasons[entry.task].length > 0); + const selected = mode === "shadow" ? entries : affected; + return [job, { run: selected.length > 0, matrix: { include: selected.length ? selected : entries }, + selected: selected.length, proposed: affected.length, total: entries.length }]; + })); + return { version: 1, mode, forceReason, changed, reasons, jobs }; +} + +export function planSummary(plan) { + const rows = Object.entries(plan.jobs).map(([job, lane]) => + `| ${job} | ${lane.proposed}/${lane.total} | ${lane.selected}/${lane.total} |`); + return ["### Incremental CI", "", `Mode: **${plan.mode}**${plan.forceReason ? ` — ${plan.forceReason}` : ""}`, + "", "| Suite | Affected jobs | Scheduled jobs |", "| --- | ---: | ---: |", ...rows, + "", "
Selection reasons", "", "```json", JSON.stringify(plan.reasons, null, 2), + "```", "
", ""].join("\n"); +} + +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const config = JSON.parse(readFileSync(process.env.CI_TASK_CONFIG ?? "turbo.json", "utf8")); + const lanes = JSON.parse(readFileSync(process.env.CI_TASK_MATRIX ?? ".github/ci-task-matrix.json", "utf8")); + let paths = [], forceReason = process.env.CI_FORCE_REASON || null; + try { + if (process.env.CI_COMPARISON) { + paths = comparisonPaths(JSON.parse(readFileSync(process.env.CI_COMPARISON, "utf8"))); + } else { + const [base, head = "HEAD"] = process.argv.slice(2); + if (!base) throw new Error("Usage: ci-plan.mjs [head]"); + paths = execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", `${base}...${head}`, "--"], + { encoding: "utf8" }).split("\0").filter(Boolean); + } + } catch (error) { forceReason ??= `Diff unavailable: ${error.message}`; } + const plan = planTasks({ config, lanes, paths, mode: process.env.CI_INCREMENTAL_MODE || "enabled", forceReason }); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`); + else console.log(JSON.stringify(plan, null, 2)); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, planSummary(plan)); + } catch (error) { + console.error(`Incremental CI unavailable; selecting full CI: ${error.message}`); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, "plan=\n"); + else process.exitCode = 1; + } +} diff --git a/.github/scripts/tests/ci-plan.test.mjs b/.github/scripts/tests/ci-plan.test.mjs new file mode 100644 index 00000000000..bd50fa52559 --- /dev/null +++ b/.github/scripts/tests/ci-plan.test.mjs @@ -0,0 +1,221 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { comparisonPaths, planTasks } from "../ci-plan.mjs"; + +const root = new URL("../../../", import.meta.url); +const config = JSON.parse(readFileSync(new URL("turbo.json", root), "utf8")); +const lanes = JSON.parse(readFileSync(new URL(".github/ci-task-matrix.json", root), "utf8")); +const plan = (paths, options = {}) => planTasks({ config, lanes, paths, ...options }); +const selected = (result, job) => result.jobs[job].run; +const selectedTasks = (result) => new Set(Object.values(result.jobs).filter((job) => job.run) + .flatMap((job) => job.matrix.include.map((entry) => entry.task))); + +test("trusted workflow fallbacks preserve every declared partition and command", () => { + const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); + for (const [job, entries] of Object.entries(lanes)) { + const block = workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0]; + assert.ok(block.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job); + if (!block.includes(" matrix:")) { assert.equal(entries.length, 1); continue; } + const fallback = block.match(/matrix:.*\|\| '(.*)'\) \}\}/)?.[1]; + assert.ok(fallback, `${job}: missing full fallback`); + assert.deepEqual(JSON.parse(fallback), { include: entries }, job); + } +}); + +test("a UI component retains browser and build coverage without unrelated server/Rust jobs", () => { + const result = plan(["ui/src/components/Sidebar.tsx"]); + assert.equal(selected(result, "build"), true); + assert.equal(selected(result, "e2e_shards"), true); + assert.equal(selected(result, "verify_serialized_server"), false); + assert.deepEqual([...selectedTasks(result)].filter((task) => /server|chat|runner/.test(task)), []); + assert.equal(result.jobs.general_tests.selected, 2); +}); + +test("source inspected across package boundaries selects its consumers", () => { + for (const file of ["ui/src/lib/issue-execution-policy.ts", "ui/src/index.css", "ui/src/pages/CompanyExport.tsx", + "ui/src/components/MarkdownBody.tsx"]) { + const tasks = selectedTasks(plan([file])); + assert.ok(tasks.has("ci:server"), file); + assert.ok(tasks.has("ci:runner-vitest"), file); + } + assert.ok(selectedTasks(plan(["ui/public/app-logos/example.svg"])).has("ci:workspaces-b")); +}); + +test("repository-wide guidance guards run uncached for every scanned UI file type", () => { + const source = readFileSync(new URL("server/src/__tests__/cli-invocation-safety.test.ts", root), "utf8"); + const extensions = [...source.split("const SCAN_EXTENSIONS = new Set([")[1].split("]);")[0].matchAll(/"(\.[a-z]+)"/g)] + .map((match) => match[1]); + assert.ok(extensions.length > 0); + for (const extension of extensions) { + for (const folder of ["ui/src/components", "ui/src/pages", "ui/src/hooks", "ui/public"]) { + assert.ok(plan([`${folder}/guidance${extension}`]).jobs.typecheck_release_registry.run); + } + } + assert.equal(config.tasks["//#ci:typecheck"].cache, false); + const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); + const job = workflow.split("\n typecheck_release_registry:\n")[1].split("\n general_tests:\n")[0]; + assert.match(job, /- name: Verify repository-wide CLI guidance safety\n run: pnpm exec vitest run --project @paperclipai\/server server\/src\/__tests__\/cli-invocation-safety.test.ts/); + assert.ok(!job.includes("uses: ./.github/actions/run-ci-task")); +}); + +test("shared contracts, runtime prompts, protocol inputs and fixtures reach consumers", () => { + const shared = selectedTasks(plan(["packages/shared/src/types/issue.ts"])); + for (const task of ["ci:server", "ci:db", "ci:workspaces-a", "ci:workspaces-b", "ci:serialized", "ci:e2e"]) { + assert.ok(shared.has(task), task); + } + assert.ok(selectedTasks(plan(["server/src/onboarding-assets/default/AGENTS.md"])).has("ci:runner-vitest")); + assert.ok(selectedTasks(plan(["packages/paperclip-runner/protocol/schema.json"])).has("ci:runner-rust")); + for (const file of ["tests/runner-e2e/fixtures/prompt.md", "scripts/run-vitest-stable.mjs", "pnpm-lock.yaml"]) { + assert.ok(Object.values(plan([file]).jobs).every((job) => job.selected === job.total), file); + } +}); + +test("renames out of source and deletions cannot masquerade as release notes", () => { + const paths = comparisonPaths({ files: [ + { filename: "releases/v2026.1029.0.md", previous_filename: "packages/shared/src/constants.ts", status: "renamed" }, + { filename: "server/src/deleted.ts", status: "removed" }, + ] }); + assert.ok(selectedTasks(plan(paths)).has("ci:server")); + assert.ok(selectedTasks(plan(paths)).has("ci:db")); +}); + +test("unclassified paths and uncertain comparisons select full verification", () => { + for (const paths of [[], ["new-root/file.ts"], ["../escaped"]]) { + const result = plan(paths); + assert.ok(result.forceReason); + assert.ok(Object.values(result.jobs).every((job) => job.selected === job.total)); + } + for (const files of [null, [], [{ status: "modified" }], [{ filename: "a", status: "renamed" }], + Array.from({ length: 300 }, () => ({ filename: "ui/src/test.ts" }))]) { + assert.throws(() => comparisonPaths({ files })); + } +}); + +test("shadow mode explains savings but runs every task; force overrides selection", () => { + const shadow = plan(["ui/src/components/Sidebar.tsx"], { mode: "shadow" }); + assert.equal(shadow.jobs.verify_serialized_server.proposed, 0); + assert.equal(shadow.jobs.verify_serialized_server.selected, 9); + for (const options of [{ mode: "full" }, { forceReason: "Stack" }]) { + assert.ok(Object.values(plan(["ui/src/components/Sidebar.tsx"], options).jobs) + .every((job) => job.selected === job.total)); + } +}); + +test("cache-affecting definition changes force execution across successive PR pushes", () => { + for (const file of ["turbo.json", "package.json", "packages/db/package.json", "scripts/ci/run-task.mjs", + ".github/actions/run-ci-task/action.yml"]) { + for (const paths of [[file], [file, "packages/db/src/schema/issues.ts"]]) { + const result = plan(paths); + assert.match(result.forceReason, /Global CI input changed/); + assert.ok(Object.values(result.jobs).every((job) => job.selected === job.total)); + } + } + const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); + assert.equal((workflow.match(/force:.*fromJSON\(needs.gate.outputs.plan\).forceReason != null/g) ?? []).length, 3); + const action = readFileSync(new URL(".github/actions/run-ci-task/action.yml", root), "utf8"); + assert.match(action, /CI_FORCE_TASKS: \$\{\{ inputs.force \}\}/); + assert.equal((action.match(/if:.*inputs.force != 'true'/g) ?? []).length, 2); +}); + +test("task dependencies propagate input changes and unsupported graph syntax fails closed", () => { + const fixture = { globalDependencies: [], tasks: { + "//#build": { inputs: ["lib/**"] }, "//#test": { inputs: ["app/**"], dependsOn: ["//#build"] }, + } }; + assert.equal(planTasks({ config: fixture, lanes: { tests: [{ task: "test" }] }, paths: ["lib/index.ts"] }).jobs.tests.run, true); + fixture.tasks["//#build"].dependsOn = ["//#test"]; + assert.throws(() => planTasks({ config: fixture, lanes: { tests: [{ task: "test" }] }, paths: ["app/index.ts"] }), /Cyclic/); + assert.throws(() => plan(["ui/src/a.ts"], { mode: "typo" }), /Invalid CI mode/); +}); + +test("local planner reads a real Git rename and handles a missing revision conservatively", () => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-plan-")); + const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + try { + git("init", "-q"); git("config", "user.name", "CI Test"); git("config", "user.email", "ci@example.invalid"); + mkdirSync(path.join(directory, "packages/shared/src"), { recursive: true }); + writeFileSync(path.join(directory, "packages/shared/src/input.ts"), "original\n"); + git("add", "."); git("commit", "-qm", "base"); + const base = git("rev-parse", "HEAD").trim(); + mkdirSync(path.join(directory, "releases")); + git("mv", "packages/shared/src/input.ts", "releases/v1.md"); git("commit", "-qm", "rename"); + for (const revision of [base, "missing-revision"]) { + const result = spawnSync(process.execPath, [new URL("../ci-plan.mjs", import.meta.url).pathname, revision], { + cwd: directory, encoding: "utf8", env: { ...process.env, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "", + CI_TASK_CONFIG: new URL("turbo.json", root).pathname, CI_TASK_MATRIX: new URL(".github/ci-task-matrix.json", root).pathname }, + }); + assert.equal(result.status, 0, result.stderr); + const resultPlan = JSON.parse(result.stdout); + assert.ok(selectedTasks(resultPlan).has("ci:server")); + if (revision !== base) assert.match(resultPlan.forceReason, /Diff unavailable/); + } + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + +test("required aggregates reject omitted, failed and cancelled selected work", () => { + const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); + for (const id of ["verify", "e2e"]) { + const block = workflow.split(`\n ${id}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0]; + const code = block.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0] + .split("\n").map((line) => line.slice(10)).join("\n"); + const names = /needs: \[([^\]]+)\]/.exec(block)[1].split(", "); + const resultPlan = plan(["ui/src/components/Sidebar.tsx"]); + const results = Object.fromEntries(names.map((name) => [name, { result: + name === "gate" || name === "policy" || resultPlan.jobs[name]?.run ? "success" : "skipped" }])); + const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8", + env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(resultPlan), CI_RESULTS: JSON.stringify(results) } }); + assert.equal(run().status, 0); + for (const name of names) { + const previous = results[name].result; + for (const bad of previous === "success" ? ["failure", "cancelled", "skipped"] : ["success"]) { + results[name].result = bad; + assert.notEqual(run().status, 0, `${id}: ${name} must reject ${bad}`); + } + results[name].result = previous; + } + } +}); + +test("the real routing step uses pinned trusted definitions and defaults to full on fetch failure", () => { + const workflow = readFileSync(new URL(".github/workflows/ci-routing.yml", root), "utf8"); + const block = workflow.split(" - name: Plan affected CI tasks from trusted definitions\n")[1]; + const shell = block.split(" run: |\n")[1].split("\n").map((line) => line.slice(10)).join("\n"); + const directory = mkdtempSync(path.join(tmpdir(), "ci-routing-plan-")); + try { + const gh = path.join(directory, "gh"); + writeFileSync(gh, `#!/usr/bin/env node +const fs = require('node:fs'), path = require('node:path'); +const endpoint = process.argv.find(arg => arg.startsWith('/repos/')); +if (endpoint?.endsWith('/git/ref/heads/master')) console.log('a'.repeat(40)); +else if (endpoint?.includes('/contents/')) { + if (process.env.TEST_FETCH_FAILURE === 'true') process.exit(1); + const [file, query] = endpoint.split('/contents/')[1].split('?'); + if (query !== 'ref=' + 'a'.repeat(40)) process.exit(2); + process.stdout.write(fs.readFileSync(path.join(process.env.TEST_TRUSTED_ROOT, file))); +} else if (endpoint?.includes('/compare/')) console.log(JSON.stringify({ files: [{filename:'ui/src/components/Sidebar.tsx'}] })); +else process.exit(3); +`); + chmodSync(gh, 0o755); + for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) { + const output = path.join(directory, "output"); + writeFileSync(output, ""); + const result = spawnSync("bash", ["-c", shell], { encoding: "utf8", env: { + ...process.env, PATH: `${directory}:${process.env.PATH}`, TEST_TRUSTED_ROOT: root.pathname, + RUNNER_TEMP: directory, FULL_CI: "true", PROFILE_CI: "false", FORCE_FULL_CI: "false", + CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_MERGE_SHA: "c".repeat(40), + GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: "", ...overrides, + } }); + assert.equal(result.status, 0, result.stderr); + const value = readFileSync(output, "utf8").trim(); + if (overrides.TEST_FETCH_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan="); + else { + assert.notEqual(value, "plan=", `${JSON.stringify(overrides)}: ${result.stderr}`); + const resultPlan = JSON.parse(value.slice("plan=".length)); + assert.equal(resultPlan.jobs.general_tests.selected, overrides.PROFILE_CI || overrides.STACK_JSON ? 19 : 2); + } + } + } finally { rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index 9d2a07437f6..15fb8b2d0c2 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -147,13 +147,15 @@ for (const [name, stack, base, expected] of scopeCases) { const files = (...paths) => JSON.stringify(paths.map((filename) => ({ filename }))); -test("profile-ci overrides both middle-stack and release-notes-only skips", () => { +test("full-suite requests override both middle-stack and release-notes-only skips", () => { for (const overrides of [ { STACK_JSON: JSON.stringify({ position: 2, size: 3, base: { ref: "master" } }), PR_BASE_REF: "stack-1" }, { STACK_JSON: "null", PR_BASE_REF: "master", TEST_COMPARE_FILES: files("releases/v2026.1015.0.md") }, ]) { assert.equal(executeStep("Select PR CI scope", { ...overrides, PROFILE_CI: "false" }).full_ci, "false"); - assert.equal(executeStep("Select PR CI scope", { ...overrides, PROFILE_CI: "true" }).full_ci, "true"); + for (const request of [{ PROFILE_CI: "true" }, { FORCE_FULL_CI: "true" }, { CI_INCREMENTAL_MODE: "full" }]) { + assert.equal(executeStep("Select PR CI scope", { ...overrides, ...request }).full_ci, "true"); + } } }); const manyReleaseNotes = JSON.stringify(Array.from({ length: 300 }, (_, index) => ({ diff --git a/.github/workflows/ci-incremental-qualification.yml b/.github/workflows/ci-incremental-qualification.yml new file mode 100644 index 00000000000..ee423086c0c --- /dev/null +++ b/.github/workflows/ci-incremental-qualification.yml @@ -0,0 +1,69 @@ +name: Incremental CI qualification + +on: + pull_request: + paths: + - package.json + - turbo.json + - .github/ci-task-matrix.json + - .github/scripts/ci-plan.mjs + - .github/scripts/tests/ci-plan.test.mjs + - .github/actions/run-ci-task/** + - .github/workflows/ci-routing.yml + - .github/workflows/pr-trusted.yml + - .github/workflows/ci-incremental-qualification.yml + - scripts/ci/** + +permissions: + contents: read + +concurrency: + group: incremental-ci-proof-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + qualify: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Install isolated qualification tools + run: npm install --prefix "$RUNNER_TEMP/ci-tools" --no-audit --no-fund turbo@2.11.10 + - name: Prove planning and required checks + run: node --test .github/scripts/tests/ci-plan.test.mjs .github/scripts/tests/ci-routing.test.mjs + - name: Prove real cache reuse and invalidation + env: + PAPERCLIP_CI_TEST_TURBO: ${{ runner.temp }}/ci-tools/node_modules/.bin/turbo + run: node --test scripts/ci/cache.integration.test.mjs + - name: Check the real repository task graph + env: + CI_TURBO_BIN: ${{ runner.temp }}/ci-tools/node_modules/.bin/turbo + run: | + node --input-type=module <<'NODE' + import { readFileSync, writeFileSync } from 'node:fs'; + import { execFileSync } from 'node:child_process'; + import assert from 'node:assert/strict'; + import { executionPlan } from './scripts/ci/execution-plan.mjs'; + const config = JSON.parse(readFileSync('turbo.json', 'utf8')); + for (const task of ['ci:server', 'ci:db', 'ci:runner-rust']) { + const plan = executionPlan(config, task, []); + const file = process.env.RUNNER_TEMP + '/ci-task-config.json'; + writeFileSync(file, JSON.stringify(plan.config)); + const result = execFileSync(process.env.CI_TURBO_BIN, ['run', plan.target, '--root-turbo-json', file, + '--dry=json', '--cache-dir=.turbo/cache', '--', ...plan.args], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }); + assert.ok(JSON.parse(result).tasks.some((task) => task.taskId === plan.target)); + } + NODE + - name: Explain this pull request's plan + env: + CI_INCREMENTAL_MODE: shadow + BASE_SHA: ${{ github.event.pull_request.base.sha }} + MERGE_SHA: ${{ github.sha }} + run: node .github/scripts/ci-plan.mjs "$BASE_SHA" "$MERGE_SHA" diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index f5d1393f3af..122282b5a0a 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -8,6 +8,8 @@ on: value: ${{ jobs.gate.outputs.runner }} full_ci: value: ${{ jobs.gate.outputs.full_ci }} + plan: + value: ${{ jobs.gate.outputs.plan }} profile_ci: value: ${{ jobs.gate.outputs.profile_ci }} @@ -22,6 +24,7 @@ jobs: runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: + plan: ${{ steps.plan.outputs.plan }} runner: ${{ steps.route.outputs.runner }} full_ci: ${{ steps.scope.outputs.full_ci }} profile_ci: ${{ steps.profile.outputs.profile_ci }} @@ -256,6 +259,8 @@ jobs: EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} PROFILE_CI: ${{ steps.profile.outputs.profile_ci }} + FORCE_FULL_CI: ${{ contains(github.event.pull_request.labels.*.name, 'full-ci') }} + CI_INCREMENTAL_MODE: ${{ vars.CI_INCREMENTAL_MODE || 'enabled' }} STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }} PR_BASE_REF: ${{ github.event.pull_request.base.ref }} run: | @@ -326,11 +331,49 @@ jobs: reason='release notes only' fi - if [[ "${PROFILE_CI:-false}" == 'true' ]]; then + if [[ "${PROFILE_CI:-false}" == 'true' || "${FORCE_FULL_CI:-false}" == 'true' || "${CI_INCREMENTAL_MODE:-enabled}" == 'full' ]]; then full_ci='true' - reason='profile-ci requests the complete PR suite' + reason='profile-ci, full-ci or full mode requests the complete PR suite' fi echo "full_ci=$full_ci" >> "$GITHUB_OUTPUT" echo "::notice title=PR CI scope::$reason; full_ci=$full_ci" + + - name: Plan affected CI tasks from trusted definitions + id: plan + shell: bash + env: + GH_TOKEN: ${{ github.token }} + FULL_CI: ${{ steps.scope.outputs.full_ci }} + PROFILE_CI: ${{ steps.profile.outputs.profile_ci }} + FORCE_FULL_CI: ${{ contains(github.event.pull_request.labels.*.name, 'full-ci') }} + CI_INCREMENTAL_MODE: ${{ vars.CI_INCREMENTAL_MODE || 'enabled' }} + STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_MERGE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + fallback() { echo 'plan=' >> "$GITHUB_OUTPUT"; exit 0; } + [[ "$FULL_CI" == 'true' ]] || fallback + [[ "$EVENT_BASE_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback + trusted="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" || fallback + [[ "$trusted" =~ ^[0-9a-f]{40}$ ]] || fallback + directory="$(mktemp -d)" + trap 'rm -rf "$directory"' EXIT + for file in .github/scripts/ci-plan.mjs .github/ci-task-matrix.json turbo.json; do + gh api -H 'Accept: application/vnd.github.raw+json' \ + "/repos/paperclipai/paperclip/contents/$file?ref=$trusted" \ + > "$directory/${file##*/}" || fallback + done + gh api "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$EVENT_MERGE_SHA" \ + > "$directory/comparison.json" || fallback + if [[ "$PROFILE_CI" == 'true' || "$FORCE_FULL_CI" == 'true' ]]; then + export CI_FORCE_REASON='Full suite requested by label' + elif [[ "$STACK_JSON" != 'null' && -n "$STACK_JSON" ]]; then + export CI_FORCE_REASON='Stack verification includes lower pull requests' + fi + export CI_TASK_CONFIG="$directory/turbo.json" + export CI_TASK_MATRIX="$directory/ci-task-matrix.json" + export CI_COMPARISON="$directory/comparison.json" + node "$directory/ci-plan.mjs" || fallback diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 954756ceaa7..8765fbfb601 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -159,7 +159,7 @@ jobs: typecheck_release_registry: name: Typecheck + Release Registry needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.typecheck_release_registry.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 @@ -339,6 +339,9 @@ jobs: - name: Typecheck workspaces whose build scripts skip TypeScript run: pnpm run typecheck:build-gaps + - name: Verify repository-wide CLI guidance safety + run: pnpm exec vitest run --project @paperclipai/server server/src/__tests__/cli-invocation-safety.test.ts + - name: Verify release registry test coverage run: pnpm run test:release-registry @@ -356,100 +359,12 @@ jobs: general_tests: name: General tests (${{ matrix.group_label }}) needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.general_tests.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - # Keep twelve duration-balanced server shards. Each shard overlaps - # at most two isolated fork workers; cases within a file stay serial. - # The chat fixture keeps three independent database-backed shards. - - group: general-server-without-chat - group_label: server (1/12) - shard_index: 0 - shard_count: 12 - - group: general-server-without-chat - group_label: server (2/12) - shard_index: 1 - shard_count: 12 - - group: general-server-without-chat - group_label: server (3/12) - shard_index: 2 - shard_count: 12 - - group: general-server-without-chat - group_label: server (4/12) - shard_index: 3 - shard_count: 12 - - group: general-server-without-chat - group_label: server (5/12) - shard_index: 4 - shard_count: 12 - - group: general-server-without-chat - group_label: server (6/12) - shard_index: 5 - shard_count: 12 - - group: general-server-without-chat - group_label: server (7/12) - shard_index: 6 - shard_count: 12 - - group: general-server-without-chat - group_label: server (8/12) - shard_index: 7 - shard_count: 12 - - group: general-server-without-chat - group_label: server (9/12) - shard_index: 8 - shard_count: 12 - - group: general-server-without-chat - group_label: server (10/12) - shard_index: 9 - shard_count: 12 - - group: general-server-without-chat - group_label: server (11/12) - shard_index: 10 - shard_count: 12 - - group: general-server-without-chat - group_label: server (12/12) - shard_index: 11 - shard_count: 12 - - group: general-chat - group_label: chat (1/3) - shard_index: 0 - shard_count: 3 - - group: general-chat - group_label: chat (2/3) - shard_index: 1 - shard_count: 3 - - group: general-chat - group_label: chat (3/3) - shard_index: 2 - shard_count: 3 - # workspaces-a was the slowest check in the fully-green PR run - # 31371439296 (2026-08-10) at 319s, with the ui project's single - # vitest invocation accounting for ~224s and the paperclipai CLI - # ~37s. Two shards use Vitest's native --shard on each project's - # file list (ui: 439 files, cli: 54), bringing each job to roughly - # half the suite time (~130s + setup) without a duration manifest. - - group: general-workspaces-a - group_label: workspaces-a (1/2) - shard_index: 0 - shard_count: 2 - - group: general-workspaces-a - group_label: workspaces-a (2/2) - shard_index: 1 - shard_count: 2 - # workspaces-b became the slowest check (499-593s) once the server - # shards were rebalanced: the db project alone was 168-250s of its - # 436-513s of vitest time (PR runs 38084171761, 38083084276 and - # 38083020834, 2026-10-10). The db project now runs as its own - # workspaces-c lane; the rest of workspaces-b is ~265-275s of - # vitest time, so both jobs land near 300-350s with setup. - - group: general-workspaces-b - group_label: workspaces-b - - group: general-workspaces-c - group_label: workspaces-c + matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }} env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} @@ -516,7 +431,17 @@ jobs: pnpm install --frozen-lockfile fi + - name: Run grouped general test suites through Turbo + if: hashFiles('turbo.json', '.github/actions/run-ci-task/action.yml') != '' + uses: ./.github/actions/run-ci-task + with: + task: ${{ matrix.task }} + args: ${{ matrix.shard_count && format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) || '[]' }} + partition: ${{ strategy.job-index || 0 }} + force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }} + - name: Run grouped general test suites + if: hashFiles('turbo.json') == '' run: | if [ -n "${{ matrix.shard_count }}" ]; then pnpm test:run:general -- --group '${{ matrix.group }}' \ @@ -539,7 +464,7 @@ jobs: docker_context_integrity: name: Docker context integrity needs: gate - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.docker_context_integrity.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 15 @@ -594,77 +519,42 @@ jobs: retention-days: 14 verify: - # Preserve the legacy required-check name while the underlying work runs in parallel. name: verify if: ${{ always() }} - needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity] + needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run] runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 5 - steps: - - name: Fail if any split verify lane failed + - name: Validate every planned suite result env: FULL_CI: ${{ needs.gate.outputs.full_ci }} - POLICY_RESULT: ${{ needs.policy.result }} - TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }} - GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }} - RUNNER_VERIFICATION_RESULT: ${{ needs.verify_paperclip_runner.result }} - BUILD_RESULT: ${{ needs.build.result }} - DOCKER_CONTEXT_INTEGRITY_RESULT: ${{ needs.docker_context_integrity.result }} + CI_PLAN: ${{ needs.gate.outputs.plan }} + CI_RESULTS: ${{ toJSON(needs) }} run: | - test "$POLICY_RESULT" = "success" - case "$FULL_CI" in - true) - test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success" - test "$GENERAL_TESTS_RESULT" = "success" - test "$RUNNER_VERIFICATION_RESULT" = "success" - test "$BUILD_RESULT" = "success" - test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success" - ;; - false) - test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped" - test "$GENERAL_TESTS_RESULT" = "skipped" - test "$RUNNER_VERIFICATION_RESULT" = "skipped" - test "$BUILD_RESULT" = "skipped" - test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped" - ;; - *) - echo "Invalid full_ci decision: $FULL_CI" >&2 - exit 1 - ;; - esac + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + const results = JSON.parse(process.env.CI_RESULTS); + assert.equal(results.gate.result, 'success', 'CI routing must succeed'); + assert.equal(results.policy.result, 'success', 'Policy must succeed'); + assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required'); + const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null; + if (plan) assert.equal(plan.version, 1, 'Supported task plan required'); + for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","verify_serialized_server","canary_dry_run"]) { + if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job); + const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run); + assert.equal(results[job].result, selected ? 'success' : 'skipped', job); + } + NODE verify_paperclip_runner: name: Verify Paperclip Runner (${{ matrix.lane_label }}) needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_paperclip_runner.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - # check:all spent ~380s of its ~600s cache-warm step inside the - # runner package's vitest suite (run 35036001734, 2026-09-15); the - # Rust tests took ~160s and every remaining check ~60s combined, - # which made this single job the PR critical path once the test - # lanes were rebalanced. Split the vitest suite across two runners - # with Vitest's native --shard, the ~160s Rust tests into their own - # lane, and the remaining static checks into a fourth. The four - # lanes union to exactly check:all: check:static covers - # check:eval-kernel, check:protocol-without-vitest, and - # check:api-authority; check:runner covers the Rust half; and the - # two vitest shards cover the package vitest file list. - - lane_label: static checks - command: check:static - - lane_label: rust - command: check:runner - release_crate: true - - lane_label: vitest 1/2 - command: test:typescript:vitest --shard=1/2 - - lane_label: vitest 2/2 - command: test:typescript:vitest --shard=2/2 - release_crate: true + matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_paperclip_runner.matrix) || '{"include":[{"lane_label":"static checks","command":"check:static","task":"ci:runner-static"},{"lane_label":"rust","command":"check:runner","release_crate":true,"task":"ci:runner-rust"},{"lane_label":"vitest 1/2","command":"test:typescript:vitest --shard=1/2","task":"ci:runner-vitest"},{"lane_label":"vitest 2/2","command":"test:typescript:vitest --shard=2/2","release_crate":true,"task":"ci:runner-vitest"}]}') }} env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} @@ -849,7 +739,17 @@ jobs: if: ${{ steps.runner_release_crate.outputs.cache-hit == 'true' }} run: node scripts/runner-release-crate-cache.mjs mark-current + - name: Verify Runner Rust through Turbo + if: matrix.command == 'check:runner' && hashFiles('turbo.json') != '' + uses: ./.github/actions/run-ci-task + with: + task: ci:runner-rust + args: '[]' + partition: ${{ strategy.job-index || 0 }} + force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }} + - name: Verify Paperclip Runner + if: matrix.command != 'check:runner' || hashFiles('turbo.json') == '' # pnpm appends trailing args to the end of the script's shell chain, # so the vitest lanes' --shard lands on `vitest run`. Do not add a # `--` separator: pnpm forwards it literally and vitest would then @@ -870,7 +770,7 @@ jobs: build: name: Build needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.build.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 @@ -1122,48 +1022,12 @@ jobs: verify_serialized_server: name: Verify serialized server suites (${{ matrix.shard_label }}) needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - # Shards are balanced by recorded duration - # (scripts/serialized-shard-durations.json, refreshed 2026-09-12); - # round-robin used to cluster the heavy suites on one runner (291s - # vs 170-201s in run 32012408876, 2026-08-17). The suite total has - # since grown to ~1926s recorded: five shards ran ~390s of tests - # each while the rebalanced general/e2e lanes run ~210-260s, so - # nine shards level this lane to ~214s and keep it off the - # critical path. - - shard_index: 0 - shard_count: 9 - shard_label: 1/9 - - shard_index: 1 - shard_count: 9 - shard_label: 2/9 - - shard_index: 2 - shard_count: 9 - shard_label: 3/9 - - shard_index: 3 - shard_count: 9 - shard_label: 4/9 - - shard_index: 4 - shard_count: 9 - shard_label: 5/9 - - shard_index: 5 - shard_count: 9 - shard_label: 6/9 - - shard_index: 6 - shard_count: 9 - shard_label: 7/9 - - shard_index: 7 - shard_count: 9 - shard_label: 8/9 - - shard_index: 8 - shard_count: 9 - shard_label: 9/9 + matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }} env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} @@ -1230,7 +1094,17 @@ jobs: pnpm install --frozen-lockfile fi + - name: Run serialized server test shard through Turbo + if: hashFiles('turbo.json', '.github/actions/run-ci-task/action.yml') != '' + uses: ./.github/actions/run-ci-task + with: + task: ci:serialized + args: ${{ format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) }} + partition: ${{ strategy.job-index || 0 }} + force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }} + - name: Run serialized server test shard + if: hashFiles('turbo.json') == '' run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} - name: Upload CI profile timings @@ -1247,7 +1121,7 @@ jobs: canary_dry_run: name: Canary Dry Run needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.canary_dry_run.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 @@ -1462,48 +1336,12 @@ jobs: e2e_shards: name: e2e shard (${{ matrix.shard_label }}) needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' }} + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.e2e_shards.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 30 strategy: fail-fast: false - matrix: - include: - # The Playwright lane is pinned to workers=1 (tests/e2e/playwright.config.ts) - # because every spec shares one throwaway server and some toggle - # instance-level flags, so it can only be parallelized across runners. - # Each shard boots its own server, which keeps that isolation intact. - # The catalog has grown to ~1694s of serial spec time (mean of runs - # 35036001734 and 35024948947, 2026-09-15): with three shards and - # stale durations the worst shard ran 745s of specs while the best - # ran 277s. The former floors — chat-adapters-ui at ~442s and - # agent-chat at ~310s — are each split into two specs, so eight - # shards with refreshed durations level to ~212s each; the heaviest - # remaining single spec is chat-adapters-ui-messaging at ~242s. - - shard_index: 0 - shard_count: 8 - shard_label: 1/8 - - shard_index: 1 - shard_count: 8 - shard_label: 2/8 - - shard_index: 2 - shard_count: 8 - shard_label: 3/8 - - shard_index: 3 - shard_count: 8 - shard_label: 4/8 - - shard_index: 4 - shard_count: 8 - shard_label: 5/8 - - shard_index: 5 - shard_count: 8 - shard_label: 6/8 - - shard_index: 6 - shard_count: 8 - shard_label: 7/8 - - shard_index: 7 - shard_count: 8 - shard_label: 8/8 + matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.e2e_shards.matrix) || '{"include":[{"shard_index":0,"shard_count":8,"shard_label":"1/8","task":"ci:e2e"},{"shard_index":1,"shard_count":8,"shard_label":"2/8","task":"ci:e2e"},{"shard_index":2,"shard_count":8,"shard_label":"3/8","task":"ci:e2e"},{"shard_index":3,"shard_count":8,"shard_label":"4/8","task":"ci:e2e"},{"shard_index":4,"shard_count":8,"shard_label":"5/8","task":"ci:e2e"},{"shard_index":5,"shard_count":8,"shard_label":"6/8","task":"ci:e2e"},{"shard_index":6,"shard_count":8,"shard_label":"7/8","task":"ci:e2e"},{"shard_index":7,"shard_count":8,"shard_label":"8/8","task":"ci:e2e"}]}') }} env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} @@ -1628,27 +1466,29 @@ jobs: retention-days: 14 e2e: - # Preserve the legacy required-check name while the specs run sharded - # across the matrix above (same pattern as the `verify` aggregate). name: e2e if: ${{ always() }} needs: [gate, policy, e2e_shards] runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 5 - steps: - - name: Fail if any e2e shard failed + - name: Validate every planned suite result env: FULL_CI: ${{ needs.gate.outputs.full_ci }} - POLICY_RESULT: ${{ needs.policy.result }} - E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} + CI_PLAN: ${{ needs.gate.outputs.plan }} + CI_RESULTS: ${{ toJSON(needs) }} run: | - test "$POLICY_RESULT" = "success" - case "$FULL_CI" in - true) test "$E2E_SHARDS_RESULT" = "success" ;; - false) test "$E2E_SHARDS_RESULT" = "skipped" ;; - *) - echo "Invalid full_ci decision: $FULL_CI" >&2 - exit 1 - ;; - esac + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + const results = JSON.parse(process.env.CI_RESULTS); + assert.equal(results.gate.result, 'success', 'CI routing must succeed'); + assert.equal(results.policy.result, 'success', 'Policy must succeed'); + assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required'); + const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null; + if (plan) assert.equal(plan.version, 1, 'Supported task plan required'); + for (const job of ["e2e_shards"]) { + if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job); + const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run); + assert.equal(results[job].result, selected ? 'success' : 'skipped', job); + } + NODE diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9b66bc04368..baf5cd0a3f8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,7 +13,7 @@ permissions: jobs: ci: - if: github.event.action != 'labeled' || github.event.label.name == 'profile-ci' + if: github.event.action != 'labeled' || github.event.label.name == 'profile-ci' || github.event.label.name == 'full-ci' # Master requires CODEOWNERS review for .github/**, including this workflow. # The AWS runner group permits pr-trusted.yml@refs/heads/master. uses: paperclipai/paperclip/.github/workflows/pr-trusted.yml@master diff --git a/.gitignore b/.gitignore index fa72406e069..91d676637d4 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ test-results/ # Opt-in CI timing artifacts .ci-profile/ + +# Content-addressed CI task results +.turbo/ diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index adb060704d2..313bd129fdc 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -169,6 +169,59 @@ fails, or the diff is empty or may be truncated, the PR runs full CI. A notes-only PR stacked above other PRs also runs full CI, because its own diff does not include the lower PRs' code. +For ordinary full-scope PRs, the routing job further selects affected suites +using `turbo.json` and `.github/ci-task-matrix.json`. It downloads the planner +and both definitions from one canonical master revision, without executing PR +code. The comparison covers the tested merge commit. Both rename paths count; +failed, empty, malformed, or potentially truncated comparisons select the full +suite. Unclassified files also select full CI. Stack verification retains the +existing scope policy and runs all suites whenever that policy requires them. +Changes to global inputs, including the task graph, package scripts, workflows, +and execution helpers, force full execution and disable result-cache restore and +save for the entire PR diff. Later pushes cannot reuse results created under +changed cache definitions. + +The repository variable `CI_INCREMENTAL_MODE` accepts `enabled` (the default), +`shadow` (report selections but execute everything), or `full`. Add the `full-ci` +label to force all checks on one PR. Authorized `profile-ci` runs also execute +everything. Forced and shadow runs bypass task-result caches. Every routing run +with a valid plan publishes affected and scheduled job counts with input reasons +in its job summary. Required aggregates reject failed, cancelled, and unexpectedly +skipped selected suites; the verify aggregate includes serialized tests and the +canary dry run. +The uncached typecheck job also runs the repository-wide CLI guidance safety +guard. It still scans UI changes when unrelated server shards are omitted. + +General and serialized test suites and the standalone Rust suite use Turbo's +content-addressed cache. An execution adapter maps selected lanes to the existing +package scripts and their declared inputs. It selects tracked and non-ignored +source files, so generated build outputs do not invalidate later task results. +The test scripts keep responsibility +for their build prerequisites. GitHub Actions stores +these results in the PR's own cache scope, so a later push can reuse unchanged +tasks without giving PRs write access to master caches. Input hashes include +the relevant source trees, fixtures, scripts, manifests, resolved lockfile, +environment, toolchain, and checkout path. Unclassified tracked file contents +also enter the execution key, including changes inherited from a newer base. +Revision-stamped builds, registry smoke tests, browser suites, and Runner lanes +that perform integration checks retain their original uncached commands. The +post-merge release verification workflow continues to execute the full suite. + +Inspect a proposed selection locally with `node .github/scripts/ci-plan.mjs origin/master HEAD`. +Run the planner and workflow-contract tests with +`node --test .github/scripts/tests/ci-plan.test.mjs`, and the real cache proof +with `node --test scripts/ci/cache.integration.test.mjs` after installing dependencies. +The separate Incremental CI qualification workflow exercises candidate changes +on GitHub-hosted runners before the trusted master workflows adopt them. + +When a suite starts reading another package, asset, or runtime prompt, update +its inputs in `turbo.json`. This includes filesystem reads as well as imports. +Server tests inspect selected UI files, shared tests inspect UI logos, and Runner +static/Vitest lanes include server integration tests. Keep those dependencies +explicit. When changing shard layout, update `.github/ci-task-matrix.json` and +the full fallback matrices in `pr-trusted.yml`; the contract tests require them +to agree. Add new cache-affecting environment settings to `globalEnv`. + The routing Fleet has two warm On-Demand runners and a four-runner ceiling. Test fleets cannot consume its runner slots. Hosted test queues therefore do not block AWS runner selection. A manual dispatch of `ci-routing.yml` on master diff --git a/doc/plans/2026-10-29-incremental-ci.md b/doc/plans/2026-10-29-incremental-ci.md new file mode 100644 index 00000000000..17584ce3689 --- /dev/null +++ b/doc/plans/2026-10-29-incremental-ci.md @@ -0,0 +1,56 @@ +# Incremental PR CI + +Goal: select existing PR suites from their inputs before allocating runners, +and reuse successful deterministic task results across pushes. Keep the current +test partitions, trusted runner admission, required checks, and full post-merge +verification. + +Delivery: a reviewable PR on `codex/incremental-ci`. The planner supports +report-only, enabled, and forced-full execution. Source/configuration uncertainty +selects the full suite. Task definitions and GitHub orchestration belong to the +CI subsystem; application behavior is outside scope. + +Acceptance: + +| Case | Required evidence | +| --- | --- | +| UI component change | UI/browser/build coverage selected; independent Rust and server suites omitted | +| UI source read by server tests | Server suites selected | +| Shared contract, fixture, or shipped prompt | Every consuming suite selected | +| Rename or deletion | Both sides of renames and deleted inputs accounted for | +| Stack, profile request, unknown path, failed/truncated diff | Existing stack policy or full verification preserved | +| Repeated identical task inputs | Real Turbo execution restores the successful task result | +| Changed source, toolchain, environment, or command | Cache miss; failed tasks never reused | +| Required checks | Selected failures/cancellations and unexpected skips reject the PR | +| Candidate workflow | GitHub-hosted qualification runs candidate planner/cache tests before trusted master adopts it | + +First feedback loop: dependency-free Node tests against real Git diffs and the +planner, then a small real Turbo cache fixture. Full required repository checks +and remote CI follow once the wiring is stable. Current server tests read select +UI files; Runner static and final Vitest lanes also execute server integration +tests. These are explicit task inputs, not inferred from package names. + +Implementation: the trusted routing workflow downloads a pinned task graph and +selects matrices before runner allocation. General and serialized Vitest suites +and standalone Runner Rust tests use PR-scoped Turbo result caches. Stable +required checks validate selected and skipped jobs. Full build and +external-registry smoke tasks remain uncached because their results depend on +revision stamps or external state. + +Local evidence: repository typechecking and build passed. Planner tests cover +real Git renames, missing/truncated comparisons, cross-package source reads, +trusted definition downloads, full-suite overrides, and required-check failures. +A real Turbo fixture proves execution through the existing package scripts, +result reuse and restoration, source and argument invalidation, environment +invalidation, and failed-task retries. The execution adapter keeps task aliases +out of package manifests. +The real server partition also passed through the adapter: Turbo reported +20.022 seconds for execution and 151 milliseconds for the repeated cache hit. +These are task timings, excluding wrapper and GitHub setup overhead. + +The planner selects 18 jobs for `ui/src/components/Sidebar.tsx`, compared with the +existing 48. This is a job-count reduction; remote wall-clock savings have not +yet been measured. The serial local full-suite attempt was stopped after more +than 30 minutes in favor of the complete parallel GitHub suite. Remote PR checks +remain the final qualification gate. Activation follows merge because trusted PR execution +loads workflow and planner definitions from master. diff --git a/package.json b/package.json index abff9ed16d5..b237a56c18a 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,8 @@ "esbuild": "^0.28.2", "typescript": "^7.0.2", "vitest": "^5.0.3", - "yaml": "^2.8.1" + "yaml": "^2.8.1", + "turbo": "2.11.10" }, "engines": { "node": ">=24.11.0" diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index b6ee3e49f2c..9f16cb86763 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -57,7 +57,7 @@ function readWorkflowJobs(workflow) { function runStackScope(stack, prBaseRef) { const workflow = readFileSync(routingWorkflow, "utf8"); const match = workflow.match( - / - name: Select PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\s*$/, + / - name: Select PR CI scope[\s\S]*? run: \|\n([\s\S]*?)(?=\n - name:|$)/, ); assert.ok(match, "routing workflow must define the PR CI scope script"); const script = match[1] @@ -89,6 +89,17 @@ function runStackScope(stack, prBaseRef) { } } +function runAggregate(block, fullCi, overrides = {}) { + const code = block.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0] + .split("\n").map((line) => line.slice(10)).join("\n"); + const names = /needs: \[([^\]]+)\]/.exec(block)[1].split(", "); + const results = Object.fromEntries(names.map((name) => [name, { result: overrides[name] + ?? (name === "gate" || name === "policy" || fullCi ? "success" : "skipped") }])); + return spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8", env: { + ...process.env, FULL_CI: String(fullCi), CI_PLAN: "", CI_RESULTS: JSON.stringify(results), + } }); +} + test("the e2e shards form a complete, non-overlapping partition", () => { const specs = listE2eSpecs(); assert.ok(specs.length > 0, "expected a non-empty e2e spec set"); @@ -178,22 +189,19 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the /^ {4}needs: \[gate, (?:policy, )?e2e_shards\]$/m, "the aggregate must depend on the runner gate, optional policy gate, and shard matrix", ); - assert.match( - aggregate, - /test "\$E2E_SHARDS_RESULT" = "success"/, - "the aggregate must fail unless every shard succeeded", - ); + assert.equal(runAggregate(aggregate, true).status, 0); + assert.equal(runAggregate(aggregate, false).status, 0); + for (const result of ["failure", "cancelled", "skipped"]) { + assert.notEqual(runAggregate(aggregate, true, { e2e_shards: result }).status, 0); + } const shards = jobs.get("e2e_shards"); assert.ok(shards, "pr-trusted.yml must define the `e2e_shards` matrix job"); - const matrixEntries = [ - ...shards.matchAll( - /^ {10}- shard_index: (?\d+)\n {12}shard_count: (?\d+)\n {12}shard_label: (?\d+\/\d+)$/gm, - ), - ].map((match) => ({ - shardIndex: Number(match.groups.shardIndex), - shardCount: Number(match.groups.shardCount), - shardLabel: match.groups.shardLabel, + const matrixEntries = JSON.parse(readFileSync(path.join(repoRoot, ".github/ci-task-matrix.json"), "utf8")) + .e2e_shards.map((entry) => ({ + shardIndex: entry.shard_index, + shardCount: entry.shard_count, + shardLabel: entry.shard_label, })); assert.equal(matrixEntries.length, SHARD_COUNT, "the shard matrix must define exactly SHARD_COUNT entries"); @@ -241,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () ]) { assert.match( jobs.get(jobId), - /^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' \}\}$/m, + /^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m, `${jobId} must run only when the gate selects full CI`, ); } @@ -255,28 +263,24 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () const verify = jobs.get("verify"); assert.match( verify, - /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity\]$/m, + /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m, ); - assert.match(verify, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/); - assert.match(verify, /test "\$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"/); - assert.match(verify, /test "\$GENERAL_TESTS_RESULT" = "skipped"/); - // Runner verification must participate in the legacy aggregate required - // check, or a runner regression could be merged while `verify` succeeds. - assert.match(verify, /RUNNER_VERIFICATION_RESULT: \$\{\{ needs\.verify_paperclip_runner\.result \}\}/); - assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "success"/); - assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "skipped"/); - assert.match(verify, /test "\$BUILD_RESULT" = "skipped"/); - // Both halves of the docker-context lane's gating: the result must be - // wired into the aggregate's env AND asserted successful on full CI — - // dropping either would let `verify` pass after the lane fails. - assert.match(verify, /DOCKER_CONTEXT_INTEGRITY_RESULT: \$\{\{ needs\.docker_context_integrity\.result \}\}/); - assert.match(verify, /test "\$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success"/); - assert.match(verify, /test "\$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped"/); + assert.equal(runAggregate(verify, true).status, 0); + assert.equal(runAggregate(verify, false).status, 0); + for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner", + "build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) { + for (const result of ["failure", "cancelled", "skipped"]) { + assert.notEqual(runAggregate(verify, true, { [job]: result }).status, 0, `${job}: ${result}`); + } + if (job !== "gate" && job !== "policy") { + assert.notEqual(runAggregate(verify, false, { [job]: "success" }).status, 0, job); + } + } const e2e = jobs.get("e2e"); assert.match(e2e, /^ {4}needs: \[gate, policy, e2e_shards\]$/m); - assert.match(e2e, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/); - assert.match(e2e, /false\) test "\$E2E_SHARDS_RESULT" = "skipped"/); + assert.notEqual(runAggregate(e2e, true, { policy: "failure" }).status, 0); + assert.notEqual(runAggregate(e2e, false, { e2e_shards: "success" }).status, 0); }); test("the stacked PR scope selector runs full CI only where intended", () => { diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index 92c92f03c3d..c4dd568a72f 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -163,7 +163,10 @@ test("workspaces-c runs the db project alone and workspaces-b no longer carries assert.ok(!workspacesB.workspaceProjects.includes("@paperclipai/db"), "db must not run twice"); assert.ok(workspacesB.workspaceProjects.length > 0, "workspaces-b must still run its other projects"); for (const workflow of ["pr-trusted.yml", "release-verify.yml"]) { - const entries = matrixEntries(readFileSync(path.join(repoRoot, ".github", "workflows", workflow), "utf8")); + const entries = workflow === "pr-trusted.yml" + ? JSON.parse(readFileSync(path.join(repoRoot, ".github/ci-task-matrix.json"), "utf8")).general_tests + .map((entry) => ({ ...entry, keys: Object.keys(entry) })) + : matrixEntries(readFileSync(path.join(repoRoot, ".github", "workflows", workflow), "utf8")); // The parser must demonstrably see shard keys in this workflow, otherwise // the unsharded assertions below could pass for the wrong reason. const sharded = entries.filter(entry => entry.group === "general-workspaces-a"); @@ -427,9 +430,10 @@ test("the PR workflow wiring for the native-runner lane holds", () => { assert.match(prWorkflow, /^name: PR$/m, "renaming pr.yml silently moves the native-runner suite back into the uncached server shards"); - const trustedWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr-trusted.yml"), "utf8"); - const lanes = [...trustedWorkflow.matchAll(/command: test:typescript:vitest --shard=(\d+)\/(\d+)/g)] - .map((match) => [Number(match[1]), Number(match[2])]); + const matrix = JSON.parse(readFileSync(path.join(repoRoot, ".github/ci-task-matrix.json"), "utf8")); + const lanes = matrix.verify_paperclip_runner + .map((entry) => /test:typescript:vitest --shard=(\d+)\/(\d+)/.exec(entry.command)) + .filter(Boolean).map((match) => [Number(match[1]), Number(match[2])]); assert.ok(lanes.length > 0, "expected sharded test:typescript:vitest lanes in pr-trusted.yml"); assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "vitest lanes must agree on the shard count"); const shardCount = lanes[0][1]; diff --git a/scripts/__tests__/runner-release-crate-cache.test.mjs b/scripts/__tests__/runner-release-crate-cache.test.mjs index c19872b38c2..ac98c949fdf 100644 --- a/scripts/__tests__/runner-release-crate-cache.test.mjs +++ b/scripts/__tests__/runner-release-crate-cache.test.mjs @@ -356,7 +356,8 @@ test("every PR reader restores the crate after the dependency cache and marks it test("the Verify Paperclip Runner job restores the crate only in the lanes that build it", () => { const body = job(prWorkflow, "verify_paperclip_runner"); - const lanes = [...body.matchAll(/- lane_label: (.+)\n\s+command: (.+)\n(\s+release_crate: (true)\n)?/g)].map(([, label, , , flag]) => [label, flag === "true"]); + const lanes = JSON.parse(readFileSync(path.join(repoRoot, ".github/ci-task-matrix.json"), "utf8")) + .verify_paperclip_runner.map((entry) => [entry.lane_label, entry.release_crate === true]); assert.deepEqual(lanes, [ ["static checks", false], ["rust", true], diff --git a/scripts/ci/cache.integration.test.mjs b/scripts/ci/cache.integration.test.mjs new file mode 100644 index 00000000000..ec9b31a285b --- /dev/null +++ b/scripts/ci/cache.integration.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { executionPlan, sourceFiles } from "./execution-plan.mjs"; + +const repositoryRoot = new URL("../../", import.meta.url); +const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname; + +test("every cached lane maps to an existing package script with complete inputs", () => { + const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8")); + for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) { + const plan = executionPlan(config, name.slice(3), ["--shard-index", "0"]); + const rootTask = plan.target.startsWith("//#"); + const manifest = JSON.parse(readFileSync(new URL(rootTask ? "package.json" + : "packages/paperclip-runner/package.json", repositoryRoot), "utf8")); + const script = plan.target.split("#")[1]; + assert.equal(typeof manifest.scripts[script], "string", plan.target); + assert.ok(plan.config.tasks[plan.target].inputs.length > 0); + for (const input of plan.config.tasks[plan.target].inputs) { + const file = rootTask ? input : input.slice("$TURBO_ROOT$/".length); + assert.ok(definition.inputs.some((glob) => path.matchesGlob(file, glob)), input); + assert.doesNotMatch(file, /\/(?:node_modules|target|dist)\//); + } + assert.ok(plan.args.includes("--shard-index")); + } +}); + +test("real Turbo executes existing suite scripts and invalidates source, arguments, environment and failed tasks", () => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-turbo-")); + const write = (name, content) => writeFileSync(path.join(directory, name), content); + const json = (name, value) => write(name, JSON.stringify(value)); + try { + execFileSync("git", ["init", "-q"], { cwd: directory }); + write(".gitignore", ".turbo/\ngenerated/\nresult.txt\ncalls.txt\n"); + mkdirSync(path.join(directory, "packages/fixture"), { recursive: true }); + json("packages/fixture/package.json", { name: "fixture", private: true }); + json("package.json", { name: "cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], + scripts: { "test:run:general": "node check.cjs" } }); + json("package-lock.json", { name: "cache-proof", lockfileVersion: 3, packages: { "": { name: "cache-proof" } } }); + const definition = { globalEnv: ["CI_TASK_CACHE_SCOPE"], tasks: { + "//#ci:server": { inputs: ["*.txt", "check.cjs"], outputs: ["result.txt"], env: ["FLAVOR"], cache: true }, + } }; + const execution = () => executionPlan(definition, "ci:server", ["--shard-index", "0", "--shard-count", "12"], sourceFiles(directory)); + write("input[one].txt", "first"); + write("check.cjs", `const fs = require('node:fs'); fs.appendFileSync('calls.txt','call\\n'); const input=fs.readFileSync('input[one].txt','utf8'); if(input==='fail') process.exit(1); if(!process.argv.includes('general-server-without-chat')) process.exit(2); fs.writeFileSync('result.txt',input+process.env.FLAVOR);`); + const run = (extra = {}, args = execution().args) => { + const plan = execution(); + json("turbo.json", plan.config); + return spawnSync(turbo, ["run", plan.target, "--cache=local:rw", "--cache-dir=.turbo/cache", "--", ...args], { + cwd: directory, encoding: "utf8", env: { ...process.env, TURBO_TELEMETRY_DISABLED: "1", FLAVOR: "a", CI_TASK_CACHE_SCOPE: "node-24", ...extra }, + }); + }; + const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length; + const pass = (extra) => { const r = run(extra); assert.equal(r.status, 0, r.stdout + r.stderr); return r; }; + pass(); assert.equal(calls(), 1); + rmSync(path.join(directory, "result.txt")); + pass(); assert.equal(calls(), 1, "cached task must not execute"); + assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), "firsta"); + write("unrelated.md", "another push"); + pass(); assert.equal(calls(), 1, "unrelated input must retain the result"); + write("input[one].txt", "second"); + pass(); assert.equal(calls(), 2, "source must invalidate checks"); + pass({ FLAVOR: "b" }); assert.equal(calls(), 3); + pass({ FLAVOR: "b", CI_TASK_CACHE_SCOPE: "node-25" }); assert.equal(calls(), 4); + assert.equal(run({}, [...execution().args, "--another-argument"]).status, 0); + assert.equal(calls(), 5, "different shard arguments must not share a task result"); + write("input[one].txt", "fail"); + assert.notEqual(run().status, 0); assert.notEqual(run().status, 0); + assert.equal(calls(), 7, "a failed task must execute on every attempt"); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/scripts/ci/execution-plan.mjs b/scripts/ci/execution-plan.mjs new file mode 100644 index 00000000000..4d1357d8c8a --- /dev/null +++ b/scripts/ci/execution-plan.mjs @@ -0,0 +1,39 @@ +import { execFileSync } from "node:child_process"; +import { matchesGlob } from "node:path"; + +export function sourceFiles(cwd = process.cwd()) { + return [...new Set(execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + { cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).split("\0").filter(Boolean))].sort(); +} + +const literalGlob = (file) => file.replace(/([*?{}[\]\\])/g, "\\$1"); + +const groups = { + "ci:server": "general-server-without-chat", + "ci:chat": "general-chat", + "ci:workspaces-a": "general-workspaces-a", + "ci:workspaces-b": "general-workspaces-b", + "ci:db": "general-workspaces-c", +}; + +export function executionPlan(config, task, args, files = sourceFiles()) { + const definition = config.tasks[`//#${task}`]; + if (!definition?.cache) throw new Error(`Task is not executable through the cache: ${task}`); + let target, commandArgs; + if (groups[task]) { + target = "//#test:run:general"; + commandArgs = ["--", "--group", groups[task], ...args]; + } else if (task === "ci:serialized") { + target = "//#test:run:serialized"; + commandArgs = ["--", ...args]; + } else if (task === "ci:runner-rust") { + target = "@paperclipai/paperclip-runner#check:runner"; + commandArgs = args; + } else throw new Error(`Missing task command: ${task}`); + const select = (globs) => files.filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob); + const selectedInputs = select(definition.inputs); + const inputs = target.startsWith("//#") ? selectedInputs + : selectedInputs.map((input) => `$TURBO_ROOT$/${input}`); + return { target, args: commandArgs, config: { ...config, + globalDependencies: select(config.globalDependencies ?? []), tasks: { [target]: { ...definition, inputs } } } }; +} diff --git a/scripts/ci/run-task.mjs b/scripts/ci/run-task.mjs new file mode 100644 index 00000000000..da05e876bf2 --- /dev/null +++ b/scripts/ci/run-task.mjs @@ -0,0 +1,52 @@ +import { spawnSync, execFileSync } from "node:child_process"; +import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { matchesGlob } from "node:path"; +import { executionPlan, sourceFiles } from "./execution-plan.mjs"; + +const task = process.env.CI_TASK; +if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task"); +const args = JSON.parse(process.env.CI_TASK_ARGS || "[]"); +if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string")) throw new Error("Invalid task arguments"); +const version = (command, argv) => { + try { return execFileSync(command, argv, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); } + catch { return "unavailable"; } +}; +const config = JSON.parse(readFileSync("turbo.json", "utf8")); +const files = sourceFiles(); +const execution = executionPlan(config, task, args, files); +mkdirSync(".turbo/task-configs", { recursive: true }); +const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`; +writeFileSync(runtimeConfig, JSON.stringify(execution.config)); +const classified = [...config.globalDependencies, ...Object.values(config.tasks).flatMap((entry) => entry.inputs)]; +const unknownInputs = createHash("sha256"); +let uncacheable = false; +for (const file of files) { + if (classified.some((glob) => matchesGlob(file, glob))) continue; + unknownInputs.update(JSON.stringify(file)); + try { + if (lstatSync(file).isFile()) unknownInputs.update(readFileSync(file)); + else uncacheable = true; + } catch (error) { + if (error.code !== "ENOENT") throw error; + unknownInputs.update(""); + } +} +const scope = { + node: process.version, pnpm: version("pnpm", ["--version"]), + platform: process.platform, arch: process.arch, cwd: realpathSync(process.cwd()), + libc: process.report.getReport().header.glibcVersionRuntime ?? null, + rust: version("rustc", ["--version", "--verbose"]), + unknownInputs: unknownInputs.digest("hex"), +}; +const force = process.env.CI_FORCE_TASKS === "true" || uncacheable; +const result = spawnSync("pnpm", ["exec", "turbo", "run", execution.target, + "--root-turbo-json", runtimeConfig, "--summarize", "--cache=local:rw", "--cache-dir=.turbo/cache", + ...(process.env.CI_TASK_DRY_RUN === "true" ? ["--dry=json"] : []), + ...(force ? ["--force"] : []), "--", ...execution.args], { + stdio: "inherit", + env: { ...process.env, TURBO_TELEMETRY_DISABLED: "1", + CI_TASK_CACHE_SCOPE: createHash("sha256").update(JSON.stringify(scope)).digest("hex") }, +}); +if (result.error) throw result.error; +process.exitCode = result.status ?? 1; diff --git a/turbo.json b/turbo.json new file mode 100644 index 00000000000..dfc4e2576cc --- /dev/null +++ b/turbo.json @@ -0,0 +1,226 @@ +{ + "$schema": "https://turborepo.dev/schema.json", + "globalDependencies": [ + "*", + "**/package.json", + "pnpm-lock.yaml", + "pnpm-workspace.yaml", + ".npmrc", + "tsconfig*.json", + "vitest.config.ts", + "scripts/**", + ".github/**", + "patches/**", + "tests/**", + "doc/**", + "docs/**", + "skills/**", + ".agents/**", + "maps/**", + ".claude/**" + ], + "globalEnv": [ + "CI", + "CI_TASK_CACHE_SCOPE", + "GITHUB_WORKFLOW", + "NODE_OPTIONS", + "TZ", + "PAPERCLIP_*", + "TEST_*", + "VITE_*", + "DATABASE_URL", + "PG*", + "RUST*", + "CARGO*", + "NODE_ENV", + "XDG_*", + "LANG", + "LC_*", + "GIT_*", + "NPM_CONFIG_*", + "npm_config_*" + ], + "globalPassThroughEnv": [ + "GITHUB_*", + "RUNNER_*", + "SSH_AUTH_SOCK", + "TMPDIR", + "TMP", + "TEMP", + "HOME", + "USER", + "SHELL" + ], + "tasks": { + "//#ci:server": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/src/api/**", + "ui/src/types/**", + "ui/src/index.css", + "ui/src/bootstrapSetup.ts", + "ui/src/pages/CompanyExport.tsx", + "ui/*", + "ui/src/components/MarkdownBody.tsx" + ], + "outputs": [], + "cache": true + }, + "//#ci:chat": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/src/api/**", + "ui/src/types/**", + "ui/src/index.css", + "ui/src/bootstrapSetup.ts", + "ui/src/pages/CompanyExport.tsx", + "ui/*", + "ui/src/components/MarkdownBody.tsx" + ], + "outputs": [], + "cache": true + }, + "//#ci:workspaces-a": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": true + }, + "//#ci:workspaces-b": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/public/**" + ], + "outputs": [], + "cache": true + }, + "//#ci:db": { + "inputs": [ + "packages/**" + ], + "outputs": [], + "cache": true + }, + "//#ci:serialized": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/src/api/**", + "ui/src/types/**", + "ui/src/index.css", + "ui/src/bootstrapSetup.ts", + "ui/src/pages/CompanyExport.tsx", + "ui/*", + "ui/src/components/MarkdownBody.tsx" + ], + "outputs": [], + "cache": true + }, + "//#ci:runner-static": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/src/api/**", + "ui/src/types/**", + "ui/src/index.css", + "ui/src/bootstrapSetup.ts", + "ui/src/pages/CompanyExport.tsx", + "ui/*", + "ui/src/components/MarkdownBody.tsx" + ], + "outputs": [], + "cache": false + }, + "//#ci:runner-rust": { + "inputs": [ + "packages/paperclip-runner/**" + ], + "outputs": [], + "cache": true + }, + "//#ci:runner-vitest": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/src/lib/**", + "ui/src/api/**", + "ui/src/types/**", + "ui/src/index.css", + "ui/src/bootstrapSetup.ts", + "ui/src/pages/CompanyExport.tsx", + "ui/*", + "ui/src/components/MarkdownBody.tsx" + ], + "outputs": [], + "cache": false + }, + "//#ci:typecheck": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false + }, + "//#ci:build": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false + }, + "//#ci:docker": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false + }, + "//#ci:canary": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false + }, + "//#ci:e2e": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false + } + } +}