diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 188567eb963..0204ccbebb6 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -207,11 +207,20 @@ of exempt prose, update this policy as well as the task inputs. For ordinary source PRs (including the audited prose inputs above), a separate `Select Vitest files` job refines the general and serialized test lanes before their runners are allocated. It runs on the gate-approved test fleet, loads the real Vitest projects, and follows -imports across workspace packages. Only projects with their own source changes -are narrowed; unchanged consumer projects retain their complete coarse lane -coverage without paying for another graph traversal. Filesystem, subprocess, network, CommonJS, +imports across workspace packages. Every project in the coarse plan is assessed, +including unchanged consumers. Projects without affected tests drop out of the +matrix; an unchanged directory alone does not prove independence. Filesystem, subprocess, network, CommonJS, dynamic-loader, unresolved, and unapproved external-package consumers are retained conservatively, including indirect consumers through local helpers. +Module-relative filesystem reads add explicit dependency edges when the path and +read API can be resolved without running code. This includes directory listings +and transitive helper reads. Dynamic paths, writes, escaped filesystem APIs, +mutable or shadowed bindings, generated inputs and paths outside the checkout +remain opaque. Setup-file reads invalidate the whole project when their input +changes. Object-valued constants, including URL objects, remain opaque even without +an observed mutation; inline URL construction and immutable string constants can +be resolved. Symlink reads track both the checked-in link and its physical target. +Missing inputs retain coverage. The selector explicitly tracks setup/configuration dependencies as well as test imports. Ordinary source names such as `session-config-metadata.test.ts` or `setup-state.ts` remain eligible; the words `config` and `setup` alone do not @@ -220,6 +229,15 @@ inputs. Other resolved setup/config dependencies invalidate their entire project The small pure-dependency allowlist lives in `scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque until reviewed. This is deliberately broader than `vitest related` alone. +The reviewed Drizzle SQL/schema entry points have no ambient repository inputs. +Their full import closures are content-hashed in +`scripts/ci/reviewed-test-dependencies.mjs`; changed installed contents revoke the +exception. CommonJS and dynamic loaders reject a fingerprint rather than leave +unhashed children outside the closure. Database drivers and migration readers remain opaque. OS information, +in-memory streams, timers and compression do not read repository files themselves; +filesystem or process operations using their outputs retain their normal guards. +The syntax audit distinguishes data keys and erased types from runtime globals. +It examines the called member, not string arguments earlier in a chained call. For conservative retention, explicit complete local mock factories cut the mocked implementation's I/O edges. Partial mocks, spies, automocks, mock bypasses, and dynamic or escaped mocking APIs do not. Edits to a mocked module still use @@ -238,7 +256,8 @@ jobs; empty lanes allocate none. Serialized files remain serialized, and chat's line shards, native Runner lanes, Rust, builds, typechecking and E2E retain their existing selection/execution rules. The job summary reports files and jobs selected per lane, and the `vitest-selection` artifact records the manifest and -dependency audit. Selected jobs execute exact Vitest test specifications, so +dependency audit, including file-read edges and the first opaque dependency that +retained each test. Selected jobs execute exact Vitest test specifications, so similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards. The wrapper consumes and validates the selection before launching Vitest, then removes selection environment variables from the test process. Nested runner diff --git a/scripts/ci/repository-file-inputs.mjs b/scripts/ci/repository-file-inputs.mjs new file mode 100644 index 00000000000..b4a63744d31 --- /dev/null +++ b/scripts/ci/repository-file-inputs.mjs @@ -0,0 +1,147 @@ +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const filesystem = /^(?:node:)?fs(?:\/promises)?$/; +const readers = new Set(["readFile", "readFileSync", "existsSync", "stat", "statSync", "lstat", "lstatSync", "readdir", "readdirSync"]); + +export function repositoryFileInputs(file, program) { + if (!program.body.some(node => node.type === "ImportDeclaration" && filesystem.test(node.source.value))) { + return { covered: [], inputs: [] }; + } + const parents = new Map(), nodes = [], declarations = new Map(), constants = new Map(), imports = new Map(); + const bind = (node) => { + if (!node) return; + if (node.type === "Identifier") declarations.set(node.name, (declarations.get(node.name) ?? 0) + 1); + else if (node.type === "ObjectPattern") node.properties.forEach(p => bind(p.value ?? p.argument)); + else if (node.type === "ArrayPattern") node.elements.forEach(bind); + else if (node.type === "AssignmentPattern") bind(node.left); + else if (node.type === "RestElement") bind(node.argument); + }; + function visit(node, parent) { + if (!node || typeof node !== "object" || !node.type) return; + parents.set(node, parent); nodes.push(node); + if (node.type === "ImportDeclaration") { + for (const spec of node.specifiers) { + bind(spec.local); + const source = filesystem.test(node.source.value) ? node.source.value.replace(/^node:/, "") : node.source.value; + imports.set(spec.local.name, { source, name: spec.type === "ImportSpecifier" ? spec.imported.name : "*" }); + } + } + if (node.type === "VariableDeclarator") { + bind(node.id); + if (parent.kind === "const" && node.id.type === "Identifier") constants.set(node.id.name, node.init); + } + if (/Function/.test(node.type) || node.type === "ArrowFunctionExpression") { + bind(node.id); node.params?.forEach(bind); + } + if (node.type === "ClassDeclaration" || node.type === "ClassExpression") bind(node.id); + if (node.type === "CatchClause") bind(node.param); + if (node.type === "AssignmentExpression") bind(node.left); + if (node.type === "UpdateExpression") bind(node.argument); + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach(child => visit(child, node)); + else if (value && typeof value === "object" && value.type) visit(value, node); + } + } + visit(program); + const tainted = new Set(); + for (const node of nodes) { + if (node.type === "MetaProperty" && node.meta.name === "import" && node.property.name === "meta") { + const member = parents.get(node), use = parents.get(member); + if (member?.type !== "MemberExpression" || member.computed || !["url", "dirname", "filename"].includes(member.property.name) + || (use?.type === "AssignmentExpression" && use.left === member) + || use?.type === "UpdateExpression" || (use?.type === "UnaryExpression" && use.operator === "delete")) tainted.add("import.meta"); + } + if (node.type !== "Identifier") continue; + const parent = parents.get(node); + const binding = imports.get(node.name); + if (binding?.name === "*" && /^(?:node:)?(?:path|url)$/.test(binding.source) + && !parent?.type.startsWith("Import")) { + if (parent?.type !== "MemberExpression" || parent.object !== node + || parents.get(parent)?.type !== "CallExpression" || parents.get(parent).callee !== parent) tainted.add(node.name); + } + if (node.name === "URL" && !(parent?.type === "NewExpression" && parent.callee === node)) tainted.add("URL"); + } + const unique = (name) => declarations.get(name) === 1; + const member = (node, name) => node?.type === "MemberExpression" && !node.computed && node.property.name === name; + const importMeta = node => node?.type === "MetaProperty" && node.meta.name === "import" && node.property.name === "meta"; + function imported(node, source, name) { + if (node?.type === "Identifier" && unique(node.name) && !tainted.has(node.name)) { + const entry = imports.get(node.name); + return entry && source.test(entry.source) && entry.name === name; + } + if (member(node, name) && node.object.type === "Identifier" && unique(node.object.name) && !tainted.has(node.object.name)) { + const entry = imports.get(node.object.name); + return entry && source.test(entry.source) && entry.name === "*"; + } + return false; + } + function evaluate(node, seen = new Set()) { + if (!node) return undefined; + if (node.type === "Literal" && typeof node.value === "string") return node.value; + if (node.type === "TemplateLiteral" && !node.expressions.length) return node.quasis[0].value.cooked; + if (node.type === "Identifier" && unique(node.name) && constants.has(node.name) && !seen.has(node.name)) { + const value = evaluate(constants.get(node.name), new Set([...seen, node.name])); + return typeof value === "string" ? value : undefined; + } + if (node.type === "Identifier" && !declarations.has(node.name)) { + if (node.name === "__dirname") return path.dirname(file); + if (node.name === "__filename") return file; + } + if (node.type === "MemberExpression" && importMeta(node.object) && !node.computed && !tainted.has("import.meta")) { + if (node.property.name === "url") return pathToFileURL(file).href; + if (node.property.name === "dirname") return path.dirname(file); + if (node.property.name === "filename") return file; + } + if (node.type === "NewExpression" && node.callee.name === "URL" && !declarations.has("URL") && !tainted.has("URL") && node.arguments.length === 2) { + const args = node.arguments.map(arg => evaluate(arg, seen)); + if (args.every(arg => typeof arg === "string")) { + try { return new URL(...args); } catch { return undefined; } + } + } + if (node.type === "CallExpression") { + const args = node.arguments.map(arg => evaluate(arg, seen)); + if (imported(node.callee, /^(?:node:)?url$/, "fileURLToPath") && args.length === 1) { + try { return fileURLToPath(args[0]); } catch { return undefined; } + } + if (args.every(arg => typeof arg === "string")) { + for (const method of ["resolve", "join", "dirname"]) { + if (!imported(node.callee, /^(?:node:)?path$/, method)) continue; + if (!args.length || !path.isAbsolute(args[0]) || (method === "dirname" && args.length !== 1)) return undefined; + return path[method](...args); + } + } + } + return undefined; + } + const covered = new Set(), invalid = new Set(), inputs = new Map(); + for (const node of nodes) { + if (node.type !== "Identifier" || !filesystem.test(imports.get(node.name)?.source ?? "")) continue; + const binding = imports.get(node.name), parent = parents.get(node); + if (parent?.type.startsWith("Import")) continue; + let target = node, method = binding.name; + if (!unique(node.name)) { invalid.add(binding.source); continue; } + if (method === "*" || method === "promises") { + target = parent; + if (method === "*" && member(target, "promises")) target = parents.get(target); + if (target?.type !== "MemberExpression" || target.computed) { invalid.add(binding.source); continue; } + method = target.property.name; + } + const call = parents.get(target); + if (!readers.has(method) || call?.type !== "CallExpression" || call.callee !== target) { + invalid.add(binding.source); continue; + } + let input = evaluate(call.arguments[0]); + if (input instanceof URL) { + try { input = fileURLToPath(input); } catch { input = undefined; } + } + if (typeof input !== "string" || !path.isAbsolute(input)) { invalid.add(binding.source); continue; } + covered.add(binding.source); + const values = inputs.get(binding.source) ?? new Set(); + values.add(path.resolve(input)); inputs.set(binding.source, values); + } + return { + covered: [...covered].filter(source => !invalid.has(source)), + inputs: [...new Set([...inputs.values()].flatMap(values => [...values]))], + }; +} diff --git a/scripts/ci/reviewed-test-dependencies.mjs b/scripts/ci/reviewed-test-dependencies.mjs new file mode 100644 index 00000000000..4f782052754 --- /dev/null +++ b/scripts/ci/reviewed-test-dependencies.mjs @@ -0,0 +1,54 @@ +import { createHash } from "node:crypto"; +import { readFileSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { parseSync } from "oxc-parser"; + +const contracts = new Map([ + ["drizzle-orm/index.js", "96c38000daaf94419eddbfd168f7dc509962d6088664e96882979bf851e0dca5"], + ["drizzle-orm/pg-core/index.js", "7a6f61b63d19ff5feb4403d8f4afdba90c25364d33769c7a4b3be507c0130357"], +]); + +export function dependencyFingerprint(file, packageRoot) { + const files = new Map(); + function visit(filename) { + filename = realpathSync(filename); + if (!filename.startsWith(`${packageRoot}${path.sep}`)) throw new Error("Dependency escapes reviewed package"); + if (files.has(filename)) return; + const source = readFileSync(filename, "utf8"), ast = parseSync(filename, source); + if (ast.errors.length) throw new Error("Unparsed dependency"); + const inspect = node => { + if (!node || typeof node !== "object") return; + if (node.type === "ImportExpression" || (node.type === "Identifier" + && /^(require|createRequire|getBuiltinModule|eval|Function)$/.test(node.name))) throw new Error("Unreviewed runtime loader"); + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach(inspect); + else if (value && typeof value === "object" && value.type) inspect(value); + } + }; + inspect(ast.program); + files.set(filename, createHash("sha256").update(source).digest("hex")); + for (const node of ast.program.body) { + if (!node.source) continue; + const specifier = node.source.value; + if (!specifier.startsWith(".")) throw new Error("Unreviewed external import"); + visit(path.resolve(path.dirname(filename), specifier)); + } + } + visit(file); + return createHash("sha256").update(JSON.stringify([...files].map(([file, hash]) => + [path.relative(packageRoot, file).split(path.sep).join("/"), hash]).sort())).digest("hex"); +} + +export function dependencyReviews() { + const cache = new Map(); + return (file, relative) => { + if (!contracts.has(relative)) return false; + if (!cache.has(file)) { + try { + const packageRoot = file.slice(0, -relative.slice("drizzle-orm/".length).length - 1); + cache.set(file, dependencyFingerprint(file, packageRoot) === contracts.get(relative)); + } catch { cache.set(file, false); } + } + return cache.get(file); + }; +} diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs index c296ccd92db..5c49976f7c1 100644 --- a/scripts/ci/select-tests.mjs +++ b/scripts/ci/select-tests.mjs @@ -59,7 +59,7 @@ export async function createTestSelection(plan, head) { const graphs = []; const changed = productInputs(plan.changed); for (const project of projects) { - const graph = await selectProjectTests({ project, changed, onlyOwnChanges: true }); + const graph = await selectProjectTests({ project, changed }); graphs.push(graph); console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`); for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory; diff --git a/scripts/ci/test-selection-inputs.test.mjs b/scripts/ci/test-selection-inputs.test.mjs new file mode 100644 index 00000000000..84e809fd06a --- /dev/null +++ b/scripts/ci/test-selection-inputs.test.mjs @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { parseSync } from "oxc-parser"; +import { repositoryFileInputs } from "./repository-file-inputs.mjs"; +import { selectProjectTests, sourceDependencies } from "./vitest-selection-graph.mjs"; +import { dependencyFingerprint, dependencyReviews } from "./reviewed-test-dependencies.mjs"; + +const file = "/repo/ui/src/contract.test.ts"; +const analyze = source => repositoryFileInputs(file, parseSync(file, source).program); + +test("data properties and erased types are not global loaders or mock bypasses", () => { + const data = sourceDependencies(file, 'import {vi} from "vitest"; const schema = {arguments: {}, fetch: false, Function: "name"}; type Factory = Function;'); + assert.deepEqual(data.reasons, []); + const chained = sourceDependencies(file, 'import {z} from "zod"; const schema = z.object({message:z.literal("routes require auth; fetch input")}).refine(value => true);'); + assert.deepEqual(chained.reasons, []); + for (const source of [ + 'fetch("https://example.invalid");', + 'const load = globalThis.fetch; load("https://example.invalid");', + 'const make = globalThis.Function; make("return 1");', + 'const scope = globalThis; const make = scope.Function; make("return 1");', + 'const {Function: make} = globalThis; make("return 1");', + 'const {fetch: request} = globalThis; request("https://example.invalid");', + 'import {vi as mocker} from "vitest"; mocker.mock("node:path", () => ({join: () => "/other.ts"}));', + 'import {vi} from "vitest"; vi.mock("./input", function() { return arguments[0](); });', + 'import {vi} from "vitest"; vi.importActual("./input");', + 'const result = (fetch("https://example.invalid") as unknown);', + ]) assert.ok(sourceDependencies(file, source).reasons.length, source); +}); + +test("reviewed SQL builders exclude drivers and revoke approval when imported source changes", t => { + const root = realpathSync("server/node_modules/drizzle-orm"); + const review = dependencyReviews(); + for (const entry of ["index.js", "pg-core/index.js"]) assert.equal(review(path.join(root, entry), `drizzle-orm/${entry}`), true); + assert.equal(review(path.join(root, "postgres-js/index.js"), "drizzle-orm/postgres-js/index.js"), false); + const fixture = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-review-"))); + t.after(() => rmSync(fixture, { recursive: true, force: true })); + const entry = path.join(fixture, "index.js"), leaf = path.join(fixture, "leaf.js"); + writeFileSync(entry, 'export * from "./leaf.js";'); writeFileSync(leaf, 'export const value = 1;'); + const before = dependencyFingerprint(entry, fixture); + writeFileSync(leaf, 'export const value = 2;'); + assert.notEqual(dependencyFingerprint(entry, fixture), before); + assert.equal(dependencyReviews()(entry, "drizzle-orm/index.js"), false); + writeFileSync(leaf, 'import "node:fs";'); + assert.throws(() => dependencyFingerprint(entry, fixture), /Unreviewed external import/); + for (const source of ['import("./child.js");', 'require("./child.js");', 'const load = require; load("./child.js");']) { + writeFileSync(leaf, source); + assert.throws(() => dependencyFingerprint(entry, fixture), /Unreviewed runtime loader/); + } +}); + +test("file input edges require module-relative paths and non-escaping read APIs", () => { + for (const source of [ + 'import {readFileSync as read} from "node:fs"; read(new URL("./leaf.ts", import.meta.url));', + 'import fs from "node:fs"; import path from "node:path"; const root = path.dirname(import.meta.filename); const input = path.join(root, "leaf.ts"); fs.readFileSync(input);', + 'import {promises as fs} from "node:fs"; import {fileURLToPath} from "node:url"; fs.readFile(fileURLToPath(new URL("./leaf.ts", import.meta.url)));', + ]) { + assert.deepEqual(analyze(source), { covered: ["fs"], inputs: ["/repo/ui/src/leaf.ts"] }); + } + for (const source of [ + 'import fs from "node:fs"; fs.readFileSync(process.env.INPUT);', + 'import {readFileSync as known} from "node:fs"; import {readFileSync as unknown} from "fs"; known(new URL("./leaf.ts", import.meta.url)); unknown(process.env.INPUT);', + 'import {readFileSync as known} from "fs"; import {readFileSync as unknown} from "node:fs"; known(new URL("./leaf.ts", import.meta.url)); unknown(process.env.INPUT);', + 'import fs from "node:fs"; fs.readFileSync("src/leaf.ts");', + 'import fs from "node:fs"; const read = fs.readFileSync; read(new URL("./leaf.ts", import.meta.url));', + 'import {readFileSync as read} from "node:fs"; function load(read) { read(new URL("./leaf.ts", import.meta.url)); }', + 'import fs from "node:fs"; const input = new URL("./leaf.ts", import.meta.url); function load(input) { fs.readFileSync(input); }', + 'import fs from "node:fs"; let input = new URL("./leaf.ts", import.meta.url); fs.readFileSync(input);', + 'import fs from "node:fs"; fs.writeFileSync(new URL("./leaf.ts", import.meta.url), "value");', + 'import fs from "node:fs"; consume(fs); fs.readFileSync(new URL("./leaf.ts", import.meta.url));', + 'import fs from "node:fs"; const URL = custom; fs.readFileSync(new URL("./leaf.ts", import.meta.url));', + 'import fs from "node:fs"; fs.promises[method](new URL("./leaf.ts", import.meta.url));', + 'import fs from "node:fs"; fs.readFileSync(new URL("https://example.invalid/source.ts"));', + 'import fs from "node:fs"; const input = new URL("./leaf.ts", import.meta.url); fs.readFileSync(input);', + 'import fs from "node:fs"; const input = new URL("./leaf.ts", import.meta.url); input.pathname = "/other.ts"; fs.readFileSync(input);', + 'import fs from "node:fs"; import path from "node:path"; path.join = custom; fs.readFileSync(path.join(__dirname,"leaf.ts"));', + 'import fs from "node:fs"; import path from "node:path"; mutate(path); fs.readFileSync(path.join(__dirname,"leaf.ts"));', + 'import fs from "node:fs"; import.meta.url = "file:///other/test.ts"; fs.readFileSync(new URL("./leaf.ts", import.meta.url));', + 'import fs from "node:fs"; const meta = import.meta; meta.url = "file:///other/test.ts"; fs.readFileSync(new URL("./leaf.ts", import.meta.url));', + ]) assert.deepEqual(analyze(source).covered, [], source); +}); + +test("unchanged consumer workspaces follow cross-package imports and file inputs", async t => { + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-inputs-"))); + t.after(() => rmSync(root, { recursive: true, force: true })); + symlinkSync(path.join(process.cwd(), "node_modules"), path.join(root, "node_modules"), "dir"); + const write = (file, source) => { + mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + writeFileSync(path.join(root, file), source); + }; + write("vitest.config.mjs", 'export default {test:{projects:["consumer", "independent"]}}'); + for (const name of ["consumer", "independent"]) { + write(`${name}/vitest.config.mjs`, `export default {test:{name:"${name}",include:["*.test.ts"]}}`); + write(`${name}/unrelated.test.ts`, 'export const value = 1;'); + } + write("producer/leaf.ts", "export const value = 1;"); + write("producer/other.ts", "export const other = 1;"); + write("consumer/import.test.ts", 'import {value} from "../producer/leaf"; console.log(value);'); + write("consumer/read.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("../producer/leaf.ts",import.meta.url));'); + write("consumer/directory.test.ts", 'import {readdirSync} from "node:fs"; readdirSync(new URL("../producer/",import.meta.url));'); + write("consumer/dynamic.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(process.env.INPUT);'); + write("consumer/mixed.test.ts", 'import {readFileSync as known} from "node:fs"; import {readFileSync as unknown} from "fs"; known(new URL("../producer/leaf.ts",import.meta.url)); unknown(process.env.INPUT);'); + write("consumer/helper.ts", 'import {readFileSync} from "node:fs"; export const source = readFileSync(new URL("../producer/leaf.ts",import.meta.url));'); + write("consumer/helper.test.ts", 'import {source} from "./helper"; console.log(source);'); + write("consumer/outside.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("/etc/hosts",import.meta.url));'); + const git = (...args) => execFileSync("git", args, { cwd: root, stdio: "ignore" }); + git("init", "-q"); git("add", "."); + const select = changed => selectProjectTests({ root, project: "consumer", changed }); + const affected = await select(["producer/leaf.ts"]); + assert.deepEqual(affected.related, ["consumer/import.test.ts"]); + assert.deepEqual(affected.inputRelated, ["consumer/directory.test.ts", "consumer/helper.test.ts", "consumer/mixed.test.ts", "consumer/read.test.ts"]); + assert.deepEqual(affected.alwaysRun, ["consumer/dynamic.test.ts", "consumer/mixed.test.ts", "consumer/outside.test.ts"]); + assert.ok(!affected.selected.includes("consumer/unrelated.test.ts")); + const other = await select(["producer/other.ts"]); + assert.deepEqual(other.selected, ["consumer/directory.test.ts", "consumer/dynamic.test.ts", "consumer/mixed.test.ts", "consumer/outside.test.ts"]); + const independent = await selectProjectTests({ root, project: "independent", changed: ["producer/leaf.ts"] }); + assert.deepEqual(independent.selected, [], "a project without imports or opaque inputs can drop out entirely"); + write("producer/new.ts", "export const added = true;"); git("add", "."); + assert.ok((await select(["producer/new.ts"])).inputRelated.includes("consumer/directory.test.ts")); + symlinkSync("leaf.ts", path.join(root, "producer/link.ts")); + write("consumer/symlink.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("../producer/link.ts",import.meta.url));'); + git("add", "."); + assert.ok((await select(["producer/leaf.ts"])).inputRelated.includes("consumer/symlink.test.ts")); + rmSync(path.join(root, "producer/link.ts")); + symlinkSync("other.ts", path.join(root, "producer/link.ts")); + assert.ok((await select(["producer/link.ts"])).inputRelated.includes("consumer/symlink.test.ts")); + write("generated/unknown.ts", "export const generated = true;"); + write("consumer/generated.test.ts", 'import {readdirSync} from "node:fs"; readdirSync(new URL("../generated/",import.meta.url));'); + git("add", "consumer/generated.test.ts"); + assert.ok((await select(["producer/other.ts"])).alwaysRun.includes("consumer/generated.test.ts")); +}); diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs index 16ec7219ae0..a959dc54e8c 100644 --- a/scripts/ci/test-selection.integration.test.mjs +++ b/scripts/ci/test-selection.integration.test.mjs @@ -51,7 +51,9 @@ test("real graph retains direct and indirect filesystem, process and opaque depe const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed }); const result = await select(["ui/src/leaf.ts"]); assert.deepEqual(result.related, ["ui/src/import.test.ts"]); - for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) { + assert.ok(result.inputRelated.includes("ui/src/reader.test.ts"), "literal file reads add the missing source dependency"); + assert.ok(!result.alwaysRun.includes("ui/src/reader.test.ts")); + for (const file of ["indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) { assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`); } assert.ok(!result.selected.includes("ui/src/independent.test.ts")); @@ -73,6 +75,9 @@ test("real graph retains direct and indirect filesystem, process and opaque depe assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"), "generated inputs have no trustworthy source-to-output graph and retain their consumers"); write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));'); + const fileSetup = await select(["ui/src/leaf.ts"]); + assert.deepEqual(fileSetup.selected, fileSetup.inventory, "an explicit setup file read affects the entire project"); + write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(process.env.INPUT);'); await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph"); mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true }); for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) { diff --git a/scripts/ci/vitest-selection-graph.mjs b/scripts/ci/vitest-selection-graph.mjs index 24c42a2df80..eb071bee156 100644 --- a/scripts/ci/vitest-selection-graph.mjs +++ b/scripts/ci/vitest-selection-graph.mjs @@ -1,10 +1,12 @@ -import { readFileSync, realpathSync } from "node:fs"; +import { readFileSync, realpathSync, statSync } from "node:fs"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import path from "node:path"; import { isBuiltin } from "node:module"; import { parseSync } from "oxc-parser"; import { createVitest } from "vitest/node"; +import { repositoryFileInputs } from "./repository-file-inputs.mjs"; +import { dependencyReviews } from "./reviewed-test-dependencies.mjs"; // Everything else is opaque by default. Do not add IO clients, module loaders, // filesystem helpers or process launchers here to improve selection counts. @@ -18,7 +20,8 @@ const purePackages = new Set([ "i18next", "react-i18next", ]); const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32", - "url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]); + "url", "util", "util/types", "buffer", "events", "string_decoder", "crypto", + "os", "timers", "timers/promises", "stream", "stream/promises", "stream/web", "async_hooks", "zlib"]); // Reviewed setup performs only temporary-home IO and Supertest socket setup, // not reads of arbitrary repository inputs. Pin contents, not just paths: any // edit disables this exception until its effects are reviewed again. Continue @@ -56,6 +59,15 @@ export function sourceDependencies(file, source) { if (literal(node)) imports.add(node.value); else reasons.add("dynamic module name"); }; + const calleeName = (node) => { + if (node?.type === "Identifier") return node.name; + if (node?.type === "MetaProperty") return `${node.meta.name}.${node.property.name}`; + if (node?.type === "MemberExpression") { + const member = node.computed ? (literal(node.property) ? node.property.value : "") : node.property.name; + return `${calleeName(node.object)}.${member}`; + } + return ""; + }; for (const statement of ast.program.body) { const call = statement.type === "ExpressionStatement" ? statement.expression : null; const callee = call?.type === "CallExpression" ? call.callee : null; @@ -66,20 +78,30 @@ export function sourceDependencies(file, source) { mocks.push(call.arguments[0].value); } } - function visit(node, parent = null) { + function visit(node, parent = null, grandparent = null) { if (!node || typeof node !== "object") return; + if (node.type?.startsWith("TS") && !["TSImportEqualsDeclaration", "TSEnumDeclaration", "TSEnumMember", + "TSModuleDeclaration", "TSModuleBlock", "TSExportAssignment", "TSParameterProperty"].includes(node.type)) { + if (node.expression) visit(node.expression, node); + return; + } + const dataKey = parent && ((["Property", "ObjectProperty", "MethodDefinition", "PropertyDefinition"].includes(parent.type) + && grandparent?.type !== "ObjectPattern" && parent.key === node && !parent.computed && !parent.shorthand) + || (parent.type === "MemberExpression" && parent.property === node && !parent.computed + && !/^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name))); if ((node.type === "ImportDeclaration" && node.importKind !== "type") || (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) { if (node.source) importSource(node.source); } if (node.type === "ImportExpression") importSource(node.source); if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import"); - if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) { + if (node.type === "Identifier" && !dataKey && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) { reasons.add("external input or dynamic code"); } if ((node.type === "Identifier" && node.name === "getBuiltinModule") || (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader"); - if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name)) + if ((node.type === "Identifier" && (/^(importActual|unmock|doUnmock|doMock)$/.test(node.name) + || (mockBindings.size > 0 && node.name === "arguments" && !dataKey))) || (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass"); if (node.type === "Identifier" && mockBindings.has(node.name) && parent && parent.type !== "ImportSpecifier" @@ -87,7 +109,12 @@ export function sourceDependencies(file, source) { reasons.add("escaped or dynamic mocking API"); } if (node.type === "CallExpression") { - const name = source.slice(node.callee.start, node.callee.end); + const name = calleeName(node.callee); + if (node.callee.type === "MemberExpression" && mockBindings.has(node.callee.object.name) + && /^(mock|doMock)$/.test(node.callee.property.name) + && literal(node.arguments[0]) && /^(?:node:)?(?:fs(?:\/promises)?|path(?:\/posix|\/win32)?|url)$/.test(node.arguments[0].value)) { + reasons.add("mocked filesystem path resolution"); + } if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) { importSource(node.arguments[0]); } @@ -102,15 +129,15 @@ export function sourceDependencies(file, source) { if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code"); } for (const value of Object.values(node)) { - if (Array.isArray(value)) value.forEach((child) => visit(child, node)); - else if (value && typeof value === "object" && value.type) visit(value, node); + if (Array.isArray(value)) value.forEach((child) => visit(child, node, parent)); + else if (value && typeof value === "object" && value.type) visit(value, node, parent); } } visit(ast.program); - return { imports: [...imports], reasons: [...reasons], mocks }; + return { imports: [...imports], reasons: [...reasons], mocks, fileInputs: repositoryFileInputs(file, ast.program) }; } -export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) { +export async function selectProjectTests({ root = process.cwd(), project, changed }) { root = realpathSync(root); const started = performance.now(); const absoluteChanges = changed.map((file) => path.resolve(root, file)); @@ -123,22 +150,37 @@ export async function selectProjectTests({ root = process.cwd(), project, change const inventory = await ctx.globTestSpecifications(); if (!inventory.length) throw new Error(`Empty inventory: ${project}`); const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/"); - if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) { - // Keep coarse dependency coverage, but avoid spending a graph traversal - // on an unchanged consumer project merely to retain most of its IO tests. - const files = inventory.map(relative).sort(); - return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [], - opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 }; - } const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], { cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, }).split("\0").filter(Boolean)); const audit = new Map(); + const reviewedDependency = dependencyReviews(); const parsed = new Map(); const parse = (file) => { if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8"))); return parsed.get(file); }; + const fileInputs = new Map(); + const declaredInputs = (file, source) => { + const paths = []; + try { + for (const input of source.fileInputs.inputs) { + const resolved = realpathSync(input); + const relative = path.relative(root, resolved).split(path.sep).join("/"); + const logical = path.relative(root, input).split(path.sep).join("/"); + const directory = statSync(resolved).isDirectory(); + if (!inside(root, input) || !inside(root, resolved) || (input !== resolved && !tracked.has(logical)) || (directory + ? ![...tracked].some(file => file.startsWith(`${relative}/`)) + : !tracked.has(relative))) return []; + paths.push({ path: resolved, directory }); + if (input !== resolved) paths.push({ path: input, directory }); + } + } catch { return []; } + fileInputs.set(file, paths); + return source.fileInputs.covered; + }; + const touchesInputs = (inputs, change) => inputs.some(input => change === input.path + || (input.directory && change.startsWith(`${input.path}${path.sep}`))); const resolve = async (specifier, importer) => { try { const resolved = await environment.pluginContainer.resolveId(specifier, importer); @@ -153,12 +195,13 @@ export async function selectProjectTests({ root = process.cwd(), project, change if (setup.has(file)) return; setup.add(file); const source = parse(file); + const coveredBuiltins = declaredInputs(file, source); const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/")); const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex"); if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`); for (const specifier of source.imports) { if (isBuiltin(specifier)) { - if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`); + if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, "")) && !coveredBuiltins.includes(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`); continue; } if (reviewed && review.external.includes(specifier)) continue; @@ -175,7 +218,8 @@ export async function selectProjectTests({ root = process.cwd(), project, change else if (!resolved) throw new Error(`Unresolved setup module: ${file}`); } for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file)); - const setupChanged = absoluteChanges.some((file) => setup.has(file)); + const setupChanged = absoluteChanges.some(file => setup.has(file) + || [...setup].some(module => touchesInputs(fileInputs.get(module) ?? [], file))); const raw = await ctx.getRelevantTestSpecifications(); const modules = environment.moduleGraph.idToModuleMap; const resolvedDependencies = new Map(); @@ -192,6 +236,7 @@ export async function selectProjectTests({ root = process.cwd(), project, change for (const [id, module] of modules) { if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue; const source = parse(id); + const coveredBuiltins = declaredInputs(id, source); const reasons = [...source.reasons]; if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source"); const local = []; @@ -203,7 +248,8 @@ export async function selectProjectTests({ root = process.cwd(), project, change if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape"); for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) { if (isBuiltin(dependencyId)) { - if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`); + if (!pureBuiltins.has(dependencyId.replace(/^node:/, "")) + && !coveredBuiltins.some(source => source.replace(/^node:/, "") === dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`); continue; } const dependency = dependencyPath(dependencyId); @@ -214,7 +260,8 @@ export async function selectProjectTests({ root = process.cwd(), project, change if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`); } else { const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/"); - if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) { + if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) + || (!purePackages.has(packageName(external)) && !reviewedDependency(dependency, external))) { reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`); } } @@ -223,6 +270,8 @@ export async function selectProjectTests({ root = process.cwd(), project, change if (reasons.length) audit.set(id, [...new Set(reasons)]); } const retained = []; + const inputRelated = []; + const retentionReasons = {}; for (const spec of inventory) { const mocked = new Set(); // Honor only unconditional, explicitly supplied mock factories without an @@ -241,19 +290,28 @@ export async function selectProjectTests({ root = process.cwd(), project, change const file = queue.pop(); if (visited.has(file)) continue; visited.add(file); - if (audit.has(file)) { opaque = true; break; } + if (absoluteChanges.some(change => touchesInputs(fileInputs.get(file) ?? [], change))) inputRelated.push(spec); + if (audit.has(file)) { + retentionReasons[relative(spec)] = { module: path.relative(root, file), reasons: audit.get(file) }; + opaque = true; break; + } for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency); } if (opaque) retained.push(spec); } - const selected = [...new Set([...raw, ...retained].map(relative))].sort(); + const selected = [...new Set([...raw, ...inputRelated, ...retained].map(relative))].sort(); return { project, inventory: inventory.map(relative).sort(), selected, related: raw.map(relative).sort(), + inputRelated: [...new Set(inputRelated.map(relative))].sort(), alwaysRun: retained.map(relative).sort(), - coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])), + retentionReasons, + coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index]) + || [...fileInputs.values()].some(inputs => touchesInputs(inputs, absoluteChanges[index]))), + fileInputs: Object.fromEntries([...fileInputs].filter(([, inputs]) => inputs.length) + .map(([file, inputs]) => [path.relative(root, file), inputs.map(input => ({ ...input, path: path.relative(root, input.path) }))])), opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])), setupChanged, seconds: (performance.now() - started) / 1000,