diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7bb129325df..a3bcc6eccca 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1255,6 +1255,14 @@ graph, exact Vitest specifications, and stable test runner are shared with CI. Serialized server tests retain their owner. Native server tests remain in local coverage. Selected chat tests retain six sequential isolated line shards. +When the change touches `packages/shared/src`, `packages/plugins/sdk/src`, or an +SDK build input such as `pnpm-lock.yaml`, the command first checks the committed +SDK public input pins (`tests/runner-e2e/source-sdk-public-inputs.json`) against +the working tree. If they are stale it exits 1 and prints the refresh command: +commit the change, run +`node tests/runner-e2e/source-sdk-public-inputs.mjs $(git rev-parse HEAD) --write`, +and commit the updated registry. Otherwise the Runner support suite fails in CI. + Successful selected checks with complete input graphs are reusable in this worktree. Opaque filesystem, subprocess, network, generated/untracked-module, and unreviewed dependency consumers continue to run fresh. A selected pass never diff --git a/scripts/__tests__/local-verify.test.mjs b/scripts/__tests__/local-verify.test.mjs index 0e5e55b92c5..c60a3731b70 100644 --- a/scripts/__tests__/local-verify.test.mjs +++ b/scripts/__tests__/local-verify.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { chmodSync, cpSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -288,6 +289,49 @@ process.stdout.write(result); assert.doesNotMatch(result.text, /Local Vitest verification passed/); }); +test("stale SDK public input pins stop verification before selection; fresh pins and unrelated changes continue", async t => { + const f = fixture(t); + const sdkInputs = "tests/runner-e2e/source-sdk-public-inputs.mjs", registry = "tests/runner-e2e/source-sdk-public-inputs.json"; + const blob = text => createHash("sha1").update(`blob ${Buffer.byteLength(text)}\0`).update(text).digest("hex"); + const outputSha256 = "a".repeat(64); + f.write(sdkInputs, readFileSync(path.join(repository, sdkInputs), "utf8")); + f.write("tests/runner-e2e/report-lock.json", `${JSON.stringify({ inputSha256: "b".repeat(64), outputSha256 })}\n`); + f.write("packages/shared/src/schema.ts", "export const schema = 1;\n"); + const writeRegistry = schema => f.write(registry, `${JSON.stringify({ lockSha256: outputSha256, inputs: { + "package.json": blob(readFileSync(path.join(f.root, "package.json"), "utf8")), "packages/shared/src/schema.ts": blob(schema) } }, null, 2)}\n`); + writeRegistry("export const schema = 1;\n"); + f.git("add", "."); f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "sdk pin baseline"); + const base = f.git("rev-parse", "HEAD"); + const run = args => { + const result = spawnSync(process.execPath, [path.join(f.root, "scripts/local-verify.mjs"), "--base", base, ...args], { + cwd: f.root, env: f.env, encoding: "utf8", timeout: 120_000, maxBuffer: 8 * 1024 * 1024 }); + assert.equal(result.error, undefined); + return { ...result, text: stripVTControlCharacters(result.stdout + result.stderr) }; + }; + // Unrelated change: the pin check does not run, even with a stale registry in the tree. + f.write("ui/src/leaf.ts", "export const value = 3;\n"); + f.write("packages/shared/src/schema.ts", "export const schema = 2;\n"); + f.git("add", "packages/shared/src/schema.ts"); f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "untracked drift"); + const unrelatedBase = f.git("rev-parse", "HEAD"); + const unrelated = spawnSync(process.execPath, [path.join(f.root, "scripts/local-verify.mjs"), "--base", unrelatedBase, "--explain"], { + cwd: f.root, env: f.env, encoding: "utf8", timeout: 120_000, maxBuffer: 8 * 1024 * 1024 }); + assert.doesNotMatch(unrelated.stdout + unrelated.stderr, /is stale/); + // Pinned input changed without a registry refresh: exit 1, name the input and the command, write nothing. + const before = readFileSync(path.join(f.root, registry)); + const stale = run(["--explain"]); + assert.equal(stale.status, 1, stale.text); + assert.match(stale.text, /source-sdk-public-inputs\.json is stale for 1 pinned input\(s\): packages\/shared\/src\/schema\.ts \(changed\)/); + assert.match(stale.text, /source-sdk-public-inputs\.mjs \$\(git rev-parse HEAD\) --write/); + assert.doesNotMatch(stale.text, /Vitest test files selected|Broader verification required/); + assert.deepEqual(readFileSync(path.join(f.root, registry)), before); + // Refreshed pins: verification continues to selection. + writeRegistry("export const schema = 2;\n"); + const fresh = run(["--explain"]); + assert.doesNotMatch(fresh.text, /is stale/); + assert.match(fresh.text, /Vitest test files selected|Broader verification required/); + assert.notEqual(fresh.status, 1, fresh.text); +}); + test("documentation-only changes and empty diffs do not start a full suite", async t => { const f = fixture(t); f.write("ui/src/leaf.ts", "export const value = 1;\n"); diff --git a/scripts/local-verify.mjs b/scripts/local-verify.mjs index 09d385aa6fa..ef94f4e5fff 100644 --- a/scripts/local-verify.mjs +++ b/scripts/local-verify.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { realpathSync } from "node:fs"; -import { fileURLToPath } from "node:url"; +import { existsSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; import { createHash } from "node:crypto"; import { comparison, localEnvironment, protocolVersion, requestKey, snapshot } from "./local-verify/inputs.mjs"; @@ -16,12 +17,31 @@ function describe(plan, cached) { console.log("[verify] Local Vitest coverage does not replace required CI, typecheck/build, browser, release, or platform-specific checks."); } +// A change to packages/shared/src, the plugin SDK source, or an SDK build input must +// refresh the committed SDK public input pins, or every SDK-dependent Runner suite +// fails in CI. The check reads only the working tree, so report it before selection. +const sdkPublicInputs = "tests/runner-e2e/source-sdk-public-inputs.mjs"; +async function staleSdkPublicInputPins(root, changed) { + const module = path.join(root, sdkPublicInputs); + if (!existsSync(module)) return null; + const { isSdkPublicInput, staleSdkPublicInputsInWorkingTree, staleSdkPublicInputsMessage } = await import(pathToFileURL(module).href); + const pins = ["tests/runner-e2e/source-sdk-public-inputs.json", "tests/runner-e2e/report-lock.json"]; + if (!changed.some(file => isSdkPublicInput(file) || pins.includes(file))) return null; + const stale = await staleSdkPublicInputsInWorkingTree(root); + return stale.length ? staleSdkPublicInputsMessage(stale) : null; +} + export async function verify({ root = process.cwd(), base, full = false, explain = false, noCache = false, env = localEnvironment() } = {}) { root = realpathSync(root); const before = snapshot(root, env), diff = comparison(root, base), key = requestKey(before, diff, full); const canReuse = !noCache && before.cacheable && mayReuseAttempt(root, key); const cached = canReuse ? readPass(root, key) : null; console.log(`[verify] Base ${diff.base.slice(0, 12)}; ${diff.changed.length} changed paths, including local edits`); + const staleSdkPins = await staleSdkPublicInputPins(root, diff.changed); + if (staleSdkPins) { + console.error(`[verify] ${staleSdkPins}`); + return 1; + } if (cached) { if (snapshot(root, env).hash !== before.hash) throw new Error("Inputs changed during cache lookup. Run verification again."); describe(cached.plan, true); diff --git a/tests/runner-e2e/source-sdk-public-inputs.mjs b/tests/runner-e2e/source-sdk-public-inputs.mjs index d20306f504a..830cb86cab3 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.mjs +++ b/tests/runner-e2e/source-sdk-public-inputs.mjs @@ -94,21 +94,62 @@ export async function synchronizeSdkPublicInputs(root, revision, { write = false */ export async function staleSdkPublicInputs(root, revision = "HEAD") { const registry = JSON.parse(await readFile(path.join(root, registryPath), "utf8")); - const pinned = registry.inputs ?? {}; const committed = {}; for (const entry of git(root, ["ls-tree", "-rz", revision, "--", ...sourceRoots, ...buildInputs]).split("\0").filter(Boolean)) { const match = /^\d+ blob ([a-f0-9]{40})\t(.+)$/.exec(entry); if (match) committed[match[2]] = match[1]; } - const stale = [...new Set([...Object.keys(pinned), ...Object.keys(committed)])].sort() - .filter(file => pinned[file] !== committed[file]) - .map(file => ({ file, change: !pinned[file] ? "unpinned" : !committed[file] ? "removed" : "changed" })); // A lockfile refresh rewrites report-lock.json; lockSha256 must follow its output hash. const policy = JSON.parse(git(root, ["show", `${revision}:${reportLockPath}`])); + return compareSdkPublicInputs(registry, committed, policy); +} + +function compareSdkPublicInputs(registry, actual, policy) { + const pinned = registry.inputs ?? {}; + const stale = [...new Set([...Object.keys(pinned), ...Object.keys(actual)])].sort() + .filter(file => pinned[file] !== actual[file]) + .map(file => ({ file, change: !pinned[file] ? "unpinned" : !actual[file] ? "removed" : "changed" })); if (registry.lockSha256 !== policy.outputSha256) stale.push({ file: `${reportLockPath} outputSha256`, change: "lockSha256 changed" }); return stale; } +const committedLockBlob = root => { + try { return git(root, ["rev-parse", "--verify", "--quiet", "HEAD:pnpm-lock.yaml"]).trim(); } + catch { return null; } +}; + +/** True when a repository path is one of the inputs the registry pins. */ +export const isSdkPublicInput = file => sourcePath(file) || buildInputs.includes(file); + +/** + * The same drift guard over the working tree (tracked and untracked, non-ignored + * files), so local verification reports a missing registry refresh before the + * change is pushed. CI runs the committed-tree guard in the Runner support suite. + */ +export async function staleSdkPublicInputsInWorkingTree(root) { + const registry = JSON.parse(await readFile(path.join(root, registryPath), "utf8")); + const actual = {}; + const listed = git(root, ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", ...sourceRoots, ...buildInputs]); + for (const file of [...new Set(listed.split("\0").filter(Boolean))]) { + try { + const bytes = await readFile(path.join(root, file)); + // Preserve the authenticated restored-lock exception: prepare-report-lock.mjs may leave the + // reviewed output lock in the working tree while the registry keeps the committed input pin. + // It applies only as an uncommitted overlay over a committed lock that matches its pin. + actual[file] = file === "pnpm-lock.yaml" && hex64(registry.lockSha256) && sha256(bytes) === registry.lockSha256 + && committedLockBlob(root) === registry.inputs?.[file] ? registry.inputs[file] : blob(bytes); + } catch (error) { if (error.code !== "ENOENT") throw error; } + } + const policy = JSON.parse(await readFile(path.join(root, reportLockPath), "utf8")); + return compareSdkPublicInputs(registry, actual, policy); +} + +/** The actionable failure text shared by the CI drift guard and local verification. */ +export function staleSdkPublicInputsMessage(stale) { + return `${registryPath} is stale for ${stale.length} pinned input(s): ${stale.slice(0, 10).map(row => `${row.file} (${row.change})`).join(", ")}${stale.length > 10 ? ", ..." : ""}. ` + + "Commit your change, run `node tests/runner-e2e/source-sdk-public-inputs.mjs $(git rev-parse HEAD) --write`, and commit the updated registry."; +} + if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const [revision, mode] = process.argv.slice(2); if (process.argv.length !== 4 || !["--check", "--write"].includes(mode)) throw new Error("Usage: node tests/runner-e2e/source-sdk-public-inputs.mjs --check|--write"); diff --git a/tests/runner-e2e/source-sdk-public-inputs.test.mjs b/tests/runner-e2e/source-sdk-public-inputs.test.mjs index 98c3cfc68ad..9ca1a501bfc 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.test.mjs +++ b/tests/runner-e2e/source-sdk-public-inputs.test.mjs @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, test } from "vitest"; import { fileURLToPath } from "node:url"; -import { staleSdkPublicInputs, synchronizeSdkPublicInputs } from "./source-sdk-public-inputs.mjs"; +import { isSdkPublicInput, staleSdkPublicInputs, staleSdkPublicInputsInWorkingTree, staleSdkPublicInputsMessage, synchronizeSdkPublicInputs } from "./source-sdk-public-inputs.mjs"; const registry = "tests/runner-e2e/source-sdk-public-inputs.json"; const reportLock = "tests/runner-e2e/report-lock.json"; @@ -116,8 +116,35 @@ test("refuses a committed lock that the report-lock policy does not review", asy test("the committed SDK public source pins match this checkout's committed tree", async () => { const root = fileURLToPath(new URL("../..", import.meta.url)); const stale = await staleSdkPublicInputs(root); - assert.deepEqual(stale, [], `${registry} is stale for ${stale.length} pinned input(s): ${stale.slice(0, 10).map(row => `${row.file} (${row.change})`).join(", ")}${stale.length > 10 ? ", ..." : ""}. ` - + "Commit your change, run `node tests/runner-e2e/source-sdk-public-inputs.mjs $(git rev-parse HEAD) --write`, and commit the updated registry."); + assert.deepEqual(stale, [], staleSdkPublicInputsMessage(stale)); +}); +// pnpm verify reads the working tree, so an uncommitted or unpushed edit to a +// pinned input is reported locally instead of failing the Runner support suite in CI. +test("the working-tree drift guard sees uncommitted and untracked pinned inputs", async () => { + const { root, revision } = await fixture(); + await synchronizeSdkPublicInputs(root, revision, { write: true }); commit(root, "resync"); + assert.deepEqual(await staleSdkPublicInputsInWorkingTree(root), []); + await put(root, "packages/shared/src/schema.ts", "uncommitted synthetic edit\n"); + await put(root, "packages/shared/src/untracked.ts", "untracked synthetic source\n"); + await put(root, "packages/shared/src/ignored.ts", "ignored synthetic source\n"); await put(root, ".gitignore", "packages/shared/src/ignored.ts\n"); + const stale = await staleSdkPublicInputsInWorkingTree(root); + assert.deepEqual(stale, [{ file: "packages/shared/src/schema.ts", change: "changed" }, { file: "packages/shared/src/untracked.ts", change: "unpinned" }]); + assert.match(staleSdkPublicInputsMessage(stale), /packages\/shared\/src\/schema\.ts \(changed\).*source-sdk-public-inputs\.mjs \$\(git rev-parse HEAD\) --write/); + assert.deepEqual(await staleSdkPublicInputs(root), [], "the committed-tree guard is unaffected by uncommitted edits"); + git(root, "checkout", "--", "packages/shared/src/schema.ts"); + await rm(path.join(root, "packages/shared/src/untracked.ts")); + // The reviewed restored lock overlay in the working tree is not drift; any other lock edit is. + await writeFile(path.join(root, "pnpm-lock.yaml"), "synthetic authenticated restored lock"); + assert.deepEqual(await staleSdkPublicInputsInWorkingTree(root), []); + await writeFile(path.join(root, "pnpm-lock.yaml"), "synthetic unreviewed lock edit\n"); + assert.deepEqual(await staleSdkPublicInputsInWorkingTree(root), [{ file: "pnpm-lock.yaml", change: "changed" }]); + // Committing the reviewed output lock without a registry refresh is drift: CI's committed-tree guard reports it, so must this one. + await writeFile(path.join(root, "pnpm-lock.yaml"), "synthetic authenticated restored lock"); commit(root, "commit reviewed output lock"); + assert.deepEqual(await staleSdkPublicInputs(root), [{ file: "pnpm-lock.yaml", change: "changed" }]); + assert.deepEqual(await staleSdkPublicInputsInWorkingTree(root), [{ file: "pnpm-lock.yaml", change: "changed" }]); + assert.equal(isSdkPublicInput("packages/shared/src/types/heartbeat.ts"), true); + assert.equal(isSdkPublicInput("pnpm-lock.yaml"), true); + assert.equal(isSdkPublicInput("server/src/index.ts"), false); }); test("the drift guard names changed, unpinned and removed inputs and clears after a resync", async () => { const { root, revision } = await fixture(); diff --git a/tests/runner-e2e/source-sdk-reviewed-fixture.ts b/tests/runner-e2e/source-sdk-reviewed-fixture.ts index 2eabe2a4571..8019330403d 100644 --- a/tests/runner-e2e/source-sdk-reviewed-fixture.ts +++ b/tests/runner-e2e/source-sdk-reviewed-fixture.ts @@ -44,7 +44,8 @@ export async function prepareReviewedSdkReference(original:string,directory:stri if(![policy.inputSha256,policy.outputSha256].includes(sha(committed)))throw new Error("Committed SDK reference lock is not reviewed"); for(const [file,expected]of Object.entries(pins.inputs)){ const bytes=file==="pnpm-lock.yaml"?committed:await readFile(path.join(original,file)); - if(file!=="pnpm-lock.yaml"&&blob(bytes)!==expected)throw new Error("Reviewed SDK reference source changed"); + if(file!=="pnpm-lock.yaml"&&blob(bytes)!==expected)throw new Error(`Reviewed SDK reference source changed: ${file} does not match its pin in tests/runner-e2e/source-sdk-public-inputs.json. ` + +"Commit your change, run `node tests/runner-e2e/source-sdk-public-inputs.mjs $(git rev-parse HEAD) --write`, and commit the updated registry."); const target=path.join(directory,file);await mkdir(path.dirname(target),{recursive:true}); if(file==="pnpm-lock.yaml")await writeFile(target,bytes);else await copyFile(path.join(original,file),target); }