diff --git a/.greptile/rules.md b/.greptile/rules.md index 6dc0d1a259d..62ecf327a8c 100644 --- a/.greptile/rules.md +++ b/.greptile/rules.md @@ -12,3 +12,5 @@ referenced file as the source of truth rather than maintaining a separate copy in dashboard instructions. Do not add package.json scripts for new utilities, checks, benchmarks, or specialized tests. +The shared `verify` entry point is the repository-wide exception. Keep new +specialized checks as direct script invocations. diff --git a/AGENTS.md b/AGENTS.md index 800d3cc973e..7119e1fc26a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,13 +198,27 @@ ignored; never commit aggregate catalogs, refresh review fingerprints, or append a shared review journal. Use Git diffs and keep review evidence in the PR or issue. Structural checks are not production acceptance. -Default local/agent test path: +Default local/agent verification path: ```sh -pnpm test +pnpm verify ``` -This is the cheap default and only runs the Vitest suite. Browser suites stay opt-in: +This selects affected Vitest files with the same conservative dependency policy +as CI. It includes committed, staged, unstaged, and untracked changes against the +merge base with local `origin/master` (or `master`). Use `--base ` for another +base and `--explain` to inspect the selection. Passing checks with complete input +graphs can be reused within this worktree. Opaque integration checks run fresh. +Source, test, fixture, configuration, dependency, environment, and toolchain changes +invalidate matching results. Failed or interrupted attempts are never passes. + +Exit 2 means the change needs broader verification: run focused checks and let +required CI supply that coverage, or explicitly use `pnpm verify --full` for the +full local Vitest suite. Do not automatically follow exit 2 with a full local run. +Use `--no-cache` to request fresh execution without reading or saving passes. +`pnpm test` and `pnpm test:run` retain their full-suite behavior. + +Browser suites stay opt-in: ```sh pnpm test:e2e @@ -215,7 +229,17 @@ Run the browser suites only when your change touches them or when you are explic For normal issue work, run the smallest relevant verification first. Do not default to repo-wide typecheck/build/test on every heartbeat when a narrower check is enough to prove the change. -Run this full check before claiming repo work done in a PR-ready hand-off, or when the change scope is broad enough that targeted checks are not sufficient: +For a PR-ready handoff, run the relevant local checks and require the applicable +CI typecheck, build, and test checks to pass for the final tested revision. Record +their scope, revision, environment, and result links. Fresh CI can supply broad +coverage; do not duplicate a full local run merely to prepare a PR, resume a chat, +or repeat unchanged passing verification. A selected local pass does not replace +required CI, release qualification, browser acceptance, or a local/platform-specific +regression. After edits or a rebase, rerun affected checks and invalidate evidence +whose inputs or required coverage changed. + +Use broad local checks when they resolve an actual remaining gap, such as a local +platform fault or explicit full-suite qualification: ```sh pnpm -r typecheck @@ -264,7 +288,7 @@ When creating a pull request (via `gh pr create` or any other method), you **mus A change is done when all are true: 1. Behavior matches `doc/SPEC-implementation.md` -2. Typecheck, tests, and build pass +2. Relevant local verification and required CI typecheck, test, and build checks pass for the final revision, with any acceptance or platform-specific gaps resolved 3. Contracts are synced across db/shared/server/ui 4. Docs updated when behavior or commands change 5. PR description follows the [PR template](.github/PULL_REQUEST_TEMPLATE.md) with all sections filled in (including Model Used) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbb0efceceb..ae63cbc761e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,12 @@ Every PR must include a **Model Used** section specifying which AI model produce ### Tests Must Pass -All tests must pass before a PR can be merged. Run them locally first and verify CI is green after pushing. +All required checks must pass before a PR can be merged. Run relevant local checks +with `pnpm verify`, then require applicable typecheck, build, and test CI checks on +the final tested revision. Fresh CI can supply broad coverage. Run a full local +suite when it resolves a remaining local/platform-specific gap or is explicitly +required for qualification. See `AGENTS.md` and `doc/DEVELOPING.md` for result reuse +and coverage requirements. ### Telemetry Changes diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 1e2c236ded9..bdaa876787a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1054,7 +1054,63 @@ defines synchronization, permissions and evidence. After map edits, run `node scripts/check-maps.mjs` for structural/reference checks; it does not start an instance or qualify access as an agent. -Use the cheap local default unless you are specifically working on browser flows: +### Incremental local verification + +Use the local verification entry point for ordinary issue work: + +```sh +pnpm verify +pnpm verify --explain +pnpm verify --base origin/master +``` + +The command compares the actual working tree with the merge base of the supplied +local ref, including committed, staged, unstaged, deleted, renamed, and untracked +paths. The default ref is local `origin/master`, falling back to `master`; it does +not fetch. Tests read the current working-tree bytes. CI must still verify the +final committed revision. The task/input policy, conservative import and file-read +graph, exact Vitest specifications, and stable test runner are shared with CI. +Serialized server tests retain their owner. Native server tests remain in local +coverage. Selected chat tests retain six sequential isolated line shards. + +Successful selected checks with complete input graphs are reusable in this +worktree. Opaque filesystem, subprocess, network, generated/untracked-module, and +unreviewed dependency consumers continue to run fresh. A selected pass never +satisfies a full run. Non-isolated projects and uncertain selection require broader +verification. The command prints the local test scope and remaining CI lanes. + +An unchanged repeat can reuse both graph discovery and passing checks. Keys include +actual tracked and non-ignored file contents (including dirty and untracked files), +the shared audited-prose exclusions, dotenv files, lockfile, installation metadata, +dependency bindings and installed package contents, the complete effective child environment, Node/pnpm/Rust toolchains, +platform, checkout path, comparison, and selection. Values from the environment +are hashed, not written to results. External database/live-test settings and +out-of-checkout source symlinks disable reuse. Cache state lives under the ignored +`.turbo/local-verify/` directory and is not shared across worktrees or machines. +Installed package content digests are memoized against file identity, size, mode, +and nanosecond modification/change times; every check rescans the installation. +Generated Vite/cache directories are excluded. The first run after installation +reads package contents; unchanged repeats avoid reading those bytes again. +`last-run.json` retains the latest completed execution's status and scope. Failed/interrupted +attempts do not publish passes. Inputs changed during a run invalidate its result. + +Exit 2 reports a broad input, incomplete graph, or unsupported selection. It does +not silently launch the full suite or claim affected coverage passed. Use focused +checks and required CI to close that gap, or explicitly request a broad local run: + +```sh +pnpm verify --full +pnpm verify --no-cache +``` + +`--full` executes the complete existing Vitest path with fresh integration coverage. +`--no-cache` bypasses pass/plan reuse and does not save passes. A fresh failed or +interrupted attempt removes its old reusable pass. Required CI on the final tested +revision may supply broad typecheck/build/test coverage for a PR handoff; do not +repeat a full local suite solely because a chat resumed or a PR is ready. Keep +release checks and any real browser, provider, local-platform or acceptance proof. + +The full local Vitest commands remain available: ```sh pnpm test @@ -1092,7 +1148,9 @@ the development server's unbundled module graph on each navigation. Browser assertion deadlines and retries remain unchanged. Use `pnpm dev` separately when verifying Vite/HMR behavior. -For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +For normal issue work, start with the smallest targeted check that proves the change. +Use the incremental command above and required final-revision CI for PR handoff. +Run broad local checks when an unresolved risk or explicit qualification needs them. The Runner transport suite keeps attachment recovery, warm attachment, live adoption, and suspension in separate test files. Vitest can schedule these diff --git a/package.json b/package.json index 597f549b436..d05ed21d7ad 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/__tests__/local-verify.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", @@ -94,7 +94,8 @@ "connections:ingest-app-definitions": "node scripts/ingest-app-definitions.mjs", "test:lifecycle-baseline": "node tests/lifecycle-baseline/run.mjs", "test:lifecycle-baseline:support": "node --test tests/lifecycle-baseline/report.test.mjs", - "test:lifecycle-baseline:typecheck": "node scripts/typecheck-test-projects.mjs tests/lifecycle-baseline" + "test:lifecycle-baseline:typecheck": "node scripts/typecheck-test-projects.mjs tests/lifecycle-baseline", + "verify": "node --import ./cli/node_modules/tsx/dist/loader.mjs scripts/ensure-workspace-package-links.ts && node scripts/local-verify.mjs" }, "devDependencies": { "@playwright/test": "^1.62.1", diff --git a/scripts/__tests__/local-verify.test.mjs b/scripts/__tests__/local-verify.test.mjs new file mode 100644 index 00000000000..0e5e55b92c5 --- /dev/null +++ b/scripts/__tests__/local-verify.test.mjs @@ -0,0 +1,359 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { chmodSync, cpSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { stripVTControlCharacters } from "node:util"; +import { comparison, localEnvironment, snapshot } from "../local-verify/inputs.mjs"; +import { planVerification } from "../local-verify/plan.mjs"; + +const repository = fileURLToPath(new URL("../../", import.meta.url)); + +function fixture(t) { + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "local-verify-"))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const write = (file, text) => { + mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + writeFileSync(path.join(root, file), text); + }; + for (const directory of ["scripts/ci", "scripts/local-verify", ".github/scripts"]) { + cpSync(path.join(repository, directory), path.join(root, directory), { recursive: true }); + } + for (const file of ["scripts/local-verify.mjs", "scripts/run-vitest-stable.mjs", "scripts/general-server-shard.mjs", + "scripts/test-line-shard.mjs", "scripts/ci-profile/record.mjs", "turbo.json", ".github/ci-task-matrix.json"]) { + write(file, readFileSync(path.join(repository, file), "utf8")); + } + for (const directory of ["server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true }); + mkdirSync(path.join(root, "node_modules/.pnpm"), { recursive: true }); + for (const name of ["vitest", "oxc-parser", ".bin"]) { + symlinkSync(path.join(repository, "node_modules", name), path.join(root, "node_modules", name), "dir"); + } + for (const file of ["node_modules/.modules.yaml", "node_modules/.pnpm/lock.yaml"]) { + write(file, readFileSync(path.join(repository, file), "utf8")); + } + write(".gitignore", "node_modules\nnode_modules/\n.turbo/\n.env\n"); + write("package.json", JSON.stringify({ name: "local-verification-fixture", private: true, type: "module", packageManager: "pnpm@9.15.4", + scripts: { verify: "node scripts/local-verify.mjs", "test:run": "vitest run" } })); + write("vitest.config.mjs", 'export default {test:{projects:["ui","cli"]}};'); + for (const [directory, name] of [["ui", "@paperclipai/ui"], ["cli", "paperclipai"]]) { + write(`${directory}/vitest.config.mjs`, `export default {test:{name:${JSON.stringify(name)},include:["src/**/*.test.ts"]}};`); + } + write("ui/src/leaf.ts", "export const value = 1;\n"); + write("ui/src/other.ts", "export const other = 1;\n"); + write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import {value} from "./leaf"; test("affected consumer",()=>{expect(value).toBeGreaterThan(0);expect(process.env.LOCAL_VERIFY_FLAVOR).not.toBe("fail");});'); + write("ui/src/unrelated.test.ts", 'import {test,expect} from "vitest"; import {other} from "./other"; test("unrelated full-suite control",()=>expect(other).toBe(123456));'); + write("cli/src/independent.test.ts", 'import {test,expect} from "vitest"; test("unchanged consumer project",()=>expect(1).toBe(1));'); + const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + git("init", "-q"); git("add", "."); + git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "baseline"); + const base = git("rev-parse", "HEAD"); + write("ui/src/leaf.ts", "export const value = 2;\n"); + const env = localEnvironment({ ...process.env, FORCE_COLOR: "0" }); + delete env.NODE_TEST_CONTEXT; + delete env.DATABASE_URL; + delete env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; + for (const name of Object.keys(env)) if (/^PAPERCLIP_.*_TEST_DATABASE_URL$/.test(name)) delete env[name]; + const run = (args = [], extra = {}) => { + const result = spawnSync(process.execPath, [path.join(root, "scripts/local-verify.mjs"), "--base", base, ...args], { + cwd: root, env: { ...env, ...extra }, encoding: "utf8", timeout: 120_000, maxBuffer: 8 * 1024 * 1024, + }); + assert.equal(result.error, undefined); + return { ...result, text: stripVTControlCharacters(result.stdout + result.stderr) }; + }; + return { root, write, git, base, env, run }; +} + +test("real selected execution reuses passes and invalidates source, environment, untracked tests and broad inputs", { timeout: 180_000 }, async t => { + const f = fixture(t); + const first = f.run(); + assert.equal(first.status, 0, first.text); + assert.match(first.text, /Test Files\s+1 passed/); + assert.match(first.text, /1\/3 Vitest test files selected/); + assert.doesNotMatch(first.text, /unrelated full-suite control/); + const reused = f.run(); + assert.equal(reused.status, 0, reused.text); + assert.match(reused.text, /reusing a passing local result/); + assert.doesNotMatch(reused.text, /Test Files/); + const last = JSON.parse(readFileSync(path.join(f.root, ".turbo/local-verify/last-run.json"), "utf8")); + const passFile = path.join(f.root, ".turbo/local-verify", `${last.key}.json`); + const corrupt = JSON.parse(readFileSync(passFile, "utf8")); + corrupt.plan.selectedFiles = []; + writeFileSync(passFile, JSON.stringify(corrupt)); + const recovered = f.run(); + assert.equal(recovered.status, 0, recovered.text); + assert.doesNotMatch(recovered.text, /reusing a passing local result/); + assert.match(recovered.text, /Reusing passed general-workspaces-a-reusable/); + f.write("ui/src/leaf.ts", "export const value = 3;\n"); + const edited = f.run(); + assert.equal(edited.status, 0, edited.text); + assert.match(edited.text, /Test Files\s+1 passed/); + const environment = f.run([], { LOCAL_VERIFY_FLAVOR: "different" }); + assert.equal(environment.status, 0, environment.text); + assert.match(environment.text, /Test Files\s+1 passed/); + const environmentFailure = f.run([], { LOCAL_VERIFY_FLAVOR: "fail" }); + assert.equal(environmentFailure.status, 1, environmentFailure.text); + assert.match(environmentFailure.text, /affected consumer/); + const installed = readFileSync(path.join(f.root, "node_modules/.modules.yaml"), "utf8"); + f.write("node_modules/.modules.yaml", `${installed}\n# changed installation\n`); + const dependencies = f.run(); + assert.equal(dependencies.status, 0, dependencies.text); + assert.match(dependencies.text, /Test Files\s+1 passed/); + rmSync(path.join(f.root, "node_modules/vitest")); + const brokenInstall = f.run(); + assert.equal(brokenInstall.status, 2, brokenInstall.text); + assert.doesNotMatch(brokenInstall.text, /reusing a passing local result/); + symlinkSync(path.join(repository, "node_modules/vitest"), path.join(f.root, "node_modules/vitest"), "dir"); + f.write("ui/src/new.test.ts", 'import {test,expect} from "vitest"; test("new untracked regression",()=>expect(false).toBe(true));'); + const untracked = f.run(); + assert.equal(untracked.status, 1, untracked.text); + assert.match(untracked.text, /new untracked regression/); + const failedAgain = f.run(); + assert.equal(failedAgain.status, 1, failedAgain.text); + assert.doesNotMatch(failedAgain.text, /reusing a passing local result/); + rmSync(path.join(f.root, "ui/src/new.test.ts")); + f.write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.ts"],testTimeout:10000}};'); + const config = f.run(); + assert.equal(config.status, 2, config.text); + assert.match(config.text, /Broader verification required/); + assert.doesNotMatch(config.text, /Test Files/); + assert.match(config.text, /vitest.config.mjs/); +}); + +test("explicit full execution cannot reuse a passing subset; opaque checks stay fresh", { timeout: 180_000 }, async t => { + const f = fixture(t); + assert.equal(f.run().status, 0); + const full = f.run(["--full"]); + assert.equal(full.status, 1, full.text); + assert.match(full.text, /unrelated full-suite control/); + f.write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; import {value} from "./leaf"; test("opaque fresh check",()=>expect(readFileSync(process.env.LOCAL_VERIFY_INPUT,"utf8")).toBe(String(value)));'); + f.git("add", "ui/src/consumer.test.ts"); + f.write(".turbo/external-input", "2"); + const input = { LOCAL_VERIFY_INPUT: path.join(f.root, ".turbo/external-input") }; + const opaque = f.run([], input); + assert.equal(opaque.status, 0, opaque.text); + assert.match(opaque.text, /Test Files\s+1 passed/); + const fresh = f.run([], input); + assert.equal(fresh.status, 0, fresh.text); + assert.match(fresh.text, /Test Files\s+1 passed/); + f.write(".turbo/external-input", "changed externally"); + const external = f.run([], input); + assert.equal(external.status, 1, external.text); + assert.match(external.text, /opaque fresh check/); +}); + +test("working tree comparison includes commits, staging, deletion, rename and untracked files", t => { + const f = fixture(t); + f.git("add", "ui/src/leaf.ts"); + f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "committed change"); + f.git("mv", "ui/src/other.ts", "ui/src/renamed.ts"); + rmSync(path.join(f.root, "cli/src/independent.test.ts")); + f.write("ui/src/added.ts", "export const added = true;"); + assert.deepEqual(comparison(f.root, f.base).changed, ["cli/src/independent.test.ts", "ui/src/added.ts", "ui/src/leaf.ts", "ui/src/other.ts", "ui/src/renamed.ts"]); + assert.throws(() => comparison(f.root, "missing-ref"), /Cannot resolve a base/); +}); + +test("dependency installation, ignored dotenv and child environment enter the cache fingerprint", t => { + const f = fixture(t), before = snapshot(f.root, f.env).hash; + f.write(".env", "FEATURE=true\n"); + assert.notEqual(snapshot(f.root, f.env).hash, before); + rmSync(path.join(f.root, ".env")); + assert.equal(snapshot(f.root, f.env).hash, before); + f.write("pnpm-lock.yaml", "lockfileVersion: '9.0'\n"); + assert.notEqual(snapshot(f.root, f.env).hash, before); + assert.notEqual(snapshot(f.root, { ...f.env, NEW_TEST_FLAG: "1" }).hash, before); + assert.equal(snapshot(f.root, { ...f.env, PAPERCLIP_TEST_POSTGRES_ADMIN_URL: "postgres://external.invalid/test" }).cacheable, false); +}); + +test("installed dependency code changes invalidate a real passing result", { timeout: 120_000 }, t => { + const f = fixture(t); + const installed = "node_modules/.pnpm/clsx@1.0.0/node_modules/clsx"; + f.write(`${installed}/package.json`, '{"name":"clsx","version":"1.0.0","type":"module","exports":"./index.mjs"}'); + f.write(`${installed}/index.mjs`, "export default () => 2;\n"); + symlinkSync(path.join(f.root, installed), path.join(f.root, "node_modules/clsx"), "dir"); + f.write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import clsx from "clsx"; import {value} from "./leaf"; test("installed dependency",()=>expect(clsx()).toBe(value));'); + const first = f.run(); + assert.equal(first.status, 0, first.text); + const reused = f.run(); + assert.equal(reused.status, 0, reused.text); + assert.match(reused.text, /reusing a passing local result/); + const moduleFile = path.join(f.root, installed, "index.mjs"), original = lstatSync(moduleFile); + f.write(`${installed}/index.mjs`, "export default () => 0;\n"); + utimesSync(moduleFile, original.atime, original.mtime); + const changed = f.run(); + assert.equal(changed.status, 1, changed.text); + assert.match(changed.text, /installed dependency/); + assert.doesNotMatch(changed.text, /reusing a passing local result/); +}); + +test("repository skill directory links preserve reuse while target edits invalidate it", t => { + const f = fixture(t); + mkdirSync(path.join(f.root, ".claude/skills"), { recursive: true }); + symlinkSync("../../ui", path.join(f.root, ".claude/skills/local"), "dir"); + f.git("add", ".claude/skills/local"); + const before = snapshot(f.root, f.env); + assert.equal(before.cacheable, true); + f.write("ui/src/leaf.ts", "export const value = 3;\n"); + assert.notEqual(snapshot(f.root, f.env).hash, before.hash); +}); + +test("local chat selection schedules exactly six single-worker partitions", { timeout: 120_000 }, async t => { + const f = fixture(t); + f.write("ui/src/leaf.ts", "export const value = 1;\n"); + f.write("vitest.config.mjs", 'export default {test:{projects:["server"]}};'); + f.write("server/vitest.config.mjs", 'export default {test:{name:"@paperclipai/server",include:["src/**/*.test.ts"]}};'); + f.write("server/src/leaf.ts", "export const value = 1;\n"); + f.write("server/src/__tests__/chat-channels.integration.test.ts", 'import {test,expect} from "vitest"; import {value} from "../leaf"; test("chat selection",()=>expect(value).toBeGreaterThan(0));'); + f.write(".github/ci-task-matrix.json", JSON.stringify({ general_tests: [ + { task: "ci:server", group: "general-server-without-chat" }, { task: "ci:chat", group: "general-chat" }, + ] })); + f.git("add", "."); f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "chat baseline"); + const base = f.git("rev-parse", "HEAD"); + f.write("server/src/leaf.ts", "export const value = 2;\n"); + const plan = await planVerification({ root: f.root, env: f.env, diff: comparison(f.root, base) }); + assert.equal(plan.kind, "selected", plan.reason); + const shards = plan.jobs.flatMap(job => JSON.parse(execFileSync(process.execPath, + [path.join(f.root, "scripts/run-vitest-stable.mjs"), ...job.args, "--dry-run"], + { cwd: f.root, env: f.env, encoding: "utf8" })).chatVitestShards); + assert.deepEqual(shards, Array.from({ length: 6 }, (_, index) => ({ index, count: 6 }))); +}); + +for (const [group, task, primary, directory] of [ + ["general-workspaces-b", "ci:workspaces-b", "@paperclipai/shared", "packages/shared"], + ["general-workspaces-c", "ci:db", "@paperclipai/db", "packages/db"], +]) test(`${group} discovers and executes reusable and fresh selected partitions`, { timeout: 180_000 }, t => { + const f = fixture(t); + const inventory = JSON.parse(execFileSync(process.execPath, [path.join(f.root, "scripts/run-vitest-stable.mjs"), + "--mode", "general", "--group", group, "--dry-run"], { cwd: f.root, env: f.env, encoding: "utf8" })); + const projects = inventory.workspaceProjects.map((project, index) => { + const folder = project === primary ? directory : `packages/fixture-${index}`; + f.write(`${folder}/vitest.config.mjs`, `export default {test:{name:${JSON.stringify(project)},include:["src/**/*.test.ts"]}};`); + f.write(`${folder}/src/independent.test.ts`, 'import {test,expect} from "vitest"; test("independent workspace",()=>expect(1).toBe(1));'); + return folder; + }); + f.write("ui/src/leaf.ts", "export const value = 1;\n"); + f.write("vitest.config.mjs", `export default {test:{projects:${JSON.stringify(projects)}}};`); + f.write(".github/ci-task-matrix.json", JSON.stringify({ general_tests: [{ task, group }] })); + f.write(`${directory}/src/leaf.ts`, "export const value = 1;\n"); + f.write(`${directory}/src/reusable.test.ts`, 'import {test,expect} from "vitest"; import {value} from "./leaf"; test("workspace consumer",()=>expect(value).toBeGreaterThan(0));'); + f.write(`${directory}/src/fresh.test.ts`, 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; import {value} from "./leaf"; test("fresh workspace consumer",()=>expect(readFileSync(process.env.LOCAL_VERIFY_INPUT,"utf8")).toBe(String(value)));'); + f.git("add", "."); f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "workspace baseline"); + const base = f.git("rev-parse", "HEAD"); + f.write(`${directory}/src/leaf.ts`, "export const value = 2;\n"); + f.write(".turbo/external-input", "2"); + const env = { LOCAL_VERIFY_INPUT: path.join(f.root, ".turbo/external-input") }; + const first = f.run(["--base", base], env); + assert.equal(first.status, 0, first.text); + assert.equal((first.text.match(/Test Files\s+1 passed/g) ?? []).length, 2, first.text); + const reused = f.run(["--base", base], env); + assert.equal(reused.status, 0, reused.text); + assert.ok(reused.text.includes(`Reusing passed ${group}-reusable`), reused.text); + assert.equal((reused.text.match(/Test Files\s+1 passed/g) ?? []).length, 1, reused.text); + f.write(".turbo/external-input", "changed externally"); + const changed = f.run(["--base", base], env); + assert.equal(changed.status, 1, changed.text); + assert.match(changed.text, /fresh workspace consumer/); +}); + +test("fingerprints preserve file boundaries when content resembles another input record", t => { + const f = fixture(t), a = "ui/src/hash-a.ts", b = "ui/src/hash-b.ts"; + f.write(a, ""); f.write(b, "tail"); + const boundary = JSON.stringify(b) + String(lstatSync(path.join(f.root, b)).mode); + f.write(a, `${boundary}payload`); + const first = snapshot(f.root, f.env).hash; + f.write(a, ""); f.write(b, `payload${boundary}tail`); + assert.notEqual(snapshot(f.root, f.env).hash, first); +}); + +test("an edit between diff discovery and fingerprinting cannot certify a partial selection", { skip: process.platform === "win32" }, t => { + const f = fixture(t); + const executable = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); + f.write(".turbo/bin/package.json", '{"type":"commonjs"}'); + f.write(".turbo/bin/git", `#!${process.execPath} +const {execFileSync}=require('node:child_process'); +const fs=require('node:fs'); +const args=process.argv.slice(2); +const result=execFileSync(${JSON.stringify(executable)},args,{stdio:['ignore','pipe','pipe']}); +if(args[0]==='diff'&&!args.includes('--cached')&&!fs.existsSync('.turbo/diff-edited')){ + fs.writeFileSync('ui/src/other.ts','export const other = 2;\\n'); + fs.writeFileSync('.turbo/diff-edited','done'); +} +process.stdout.write(result); +`); + chmodSync(path.join(f.root, ".turbo/bin/git"), 0o755); + const result = f.run([], { PATH: `${path.join(f.root, ".turbo/bin")}${path.delimiter}${f.env.PATH}` }); + assert.equal(result.status, 1, result.text); + assert.match(result.text, /Inputs changed/); + assert.doesNotMatch(result.text, /Local Vitest verification passed/); +}); + +test("documentation-only changes and empty diffs do not start a full suite", async t => { + const f = fixture(t); + f.write("ui/src/leaf.ts", "export const value = 1;\n"); + f.write("doc/DEVELOPING.md", "Guidance only\n"); + const result = f.run(["--explain"]); + assert.equal(result.status, 0, result.text); + assert.match(result.text, /0 Vitest test files selected/); + assert.doesNotMatch(result.text, /Test Files/); + const plan = await planVerification({ root: f.root, env: f.env, + diff: { ...comparison(f.root, f.base), changed: [] } }); + assert.deepEqual(plan.jobs, []); +}); + +test("local server selection retains native suites and serialized ownership under a CI caller", { timeout: 120_000 }, t => { + const f = fixture(t); + f.write("ui/src/leaf.ts", "export const value = 1;\n"); + f.write("vitest.config.mjs", 'export default {test:{projects:["server"]}};'); + f.write("server/vitest.config.mjs", 'export default {test:{name:"@paperclipai/server",include:["src/**/*.test.ts"]}};'); + f.write("server/src/leaf.ts", "export const value = 1;\n"); + const native = "server/src/services/native-runtime/plugin-runner-probe.test.ts"; + const serialized = "server/src/__tests__/authz-owner.test.ts"; + for (const [file, imported] of [[native, "../../leaf"], [serialized, "../leaf"]]) { + f.write(file, `import {test,expect} from "vitest"; import {value} from ${JSON.stringify(imported)}; test("local owner",()=>expect(value).toBeGreaterThan(0));`); + } + f.write(".github/ci-task-matrix.json", JSON.stringify({ + general_tests: [{ task: "ci:server", group: "general-server-without-chat" }], + verify_serialized_server: [{ task: "ci:serialized" }], + })); + f.git("add", "."); f.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "server baseline"); + const base = f.git("rev-parse", "HEAD"); + f.write("server/src/leaf.ts", "export const value = 2;\n"); + const result = f.run(["--base", base], { GITHUB_WORKFLOW: "PR" }); + assert.equal(result.status, 0, result.text); + assert.equal((result.text.match(/Test Files\s+1 passed/g) ?? []).length, 2); + const record = JSON.parse(readFileSync(path.join(f.root, ".turbo/local-verify/last-run.json"), "utf8")); + assert.deepEqual(record.plan.manifest.lanes["general-server-without-chat"].shards[0]["@paperclipai/server"], [native]); + assert.deepEqual(record.plan.manifest.lanes.serialized.shards[0]["@paperclipai/server"], [serialized]); +}); + +test("source mutation and interruption do not publish reusable passes", { timeout: 120_000 }, async t => { + const f = fixture(t); + const runControlled = action => new Promise((resolve, reject) => { + const child = spawn(process.execPath, [path.join(f.root, "scripts/local-verify.mjs"), "--base", f.base, "--no-cache"], { + cwd: f.root, env: f.env, stdio: ["ignore", "pipe", "pipe"], + }); + let text = "", acted = false; + const timer = setTimeout(() => { child.kill("SIGTERM"); reject(new Error("Controlled verification did not finish")); }, 60_000); + const collect = bytes => { + text += bytes; + if (!acted && text.includes("[verify] Running")) { acted = true; action(child); } + }; + child.stdout.on("data", collect); child.stderr.on("data", collect); + child.once("error", reject); + child.once("close", (code, signal) => { clearTimeout(timer); resolve({ code, signal, text, acted }); }); + }); + const changed = await runControlled(() => f.write("ui/src/leaf.ts", "export const value = 4;\n")); + assert.equal(changed.acted, true); + assert.equal(changed.code, 1, changed.text); + assert.match(changed.text, /Inputs changed during verification/); + const beforeInterrupt = f.run(); + assert.equal(beforeInterrupt.status, 0, beforeInterrupt.text); + const interrupted = await runControlled(child => child.kill("SIGTERM")); + assert.equal(interrupted.acted, true); + assert.equal(interrupted.code, 143, interrupted.text); + const result = f.run(); + assert.equal(result.status, 0, result.text); + assert.match(result.text, /Test Files\s+1 passed/); + assert.doesNotMatch(result.text, /reusing a passing local result/); +}); diff --git a/scripts/ci/select-projects.mjs b/scripts/ci/select-projects.mjs index a5ef304ecf0..1bd48d008c7 100644 --- a/scripts/ci/select-projects.mjs +++ b/scripts/ci/select-projects.mjs @@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url"; // Each Vite context stays in its own process: aliases, plugins, globals and // heaps cannot leak between projects. Two contexts bound memory on hosted CI. -export async function selectProjects({ projects, changed, root = process.cwd(), concurrency = Math.min(2, availableParallelism()) }) { +export async function selectProjects({ projects, changed, root = process.cwd(), concurrency = Math.min(2, availableParallelism()), env = process.env }) { if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 2) throw new Error("Expected one or two graph workers"); const results = new Array(projects.length), children = new Set(); let next = 0, failure; @@ -13,10 +13,10 @@ export async function selectProjects({ projects, changed, root = process.cwd(), const index = next++; try { results[index] = await new Promise((resolve, reject) => { - const env = { ...process.env }; - delete env.NODE_TEST_CONTEXT; + const childEnv = { ...env }; + delete childEnv.NODE_TEST_CONTEXT; const child = fork(fileURLToPath(import.meta.url), [], { - cwd: root, env, execArgv: ["--max-old-space-size=2048"], stdio: ["ignore", "inherit", "inherit", "ipc"], + cwd: root, env: childEnv, execArgv: ["--max-old-space-size=2048"], stdio: ["ignore", "inherit", "inherit", "ipc"], }); children.add(child); let result; diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs index 16b85191615..ca488c547bc 100644 --- a/scripts/ci/select-tests.mjs +++ b/scripts/ci/select-tests.mjs @@ -8,7 +8,7 @@ import { productInputs } from "../../.github/scripts/ci-inputs.mjs"; import { mergeDiff } from "../../.github/scripts/ci-diff.mjs"; import { selectProjects } from "./select-projects.mjs"; -export function selectionEligibility(plan) { +export function selectionEligibility(plan, root = process.cwd()) { if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan"; if (!Array.isArray(plan.changed) || !plan.changed.length) return "Incomplete comparison"; const changed = productInputs(plan.changed); @@ -20,7 +20,7 @@ export function selectionEligibility(plan) { || /(?:^|\/)(?:vite|vitest)\.(?:config|setup)\.[cm]?[jt]sx?$|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`; try { - if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`; + if (!lstatSync(path.join(root, file)).isFile()) return `Non-regular source: ${file}`; } catch { return `Deleted or missing source: ${file}`; } } return null; diff --git a/scripts/ci/vitest-selection-graph.mjs b/scripts/ci/vitest-selection-graph.mjs index 78fcfb16665..6e6949ea5b0 100644 --- a/scripts/ci/vitest-selection-graph.mjs +++ b/scripts/ci/vitest-selection-graph.mjs @@ -313,6 +313,7 @@ export async function selectProjectTests({ root = process.cwd(), project, change const selected = [...new Set([...raw, ...inputRelated, ...retained].map(relative))].sort(); return { project, + isolated: owner.config.isolate !== false, inventory: inventory.map(relative).sort(), selected, related: raw.map(relative).sort(), diff --git a/scripts/local-verify.mjs b/scripts/local-verify.mjs new file mode 100644 index 00000000000..09d385aa6fa --- /dev/null +++ b/scripts/local-verify.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { createHash } from "node:crypto"; +import { comparison, localEnvironment, protocolVersion, requestKey, snapshot } from "./local-verify/inputs.mjs"; +import { planVerification } from "./local-verify/plan.mjs"; +import { invalidatePass, mayReuseAttempt, readPass, readPlan, runPlan, writeRecord } from "./local-verify/run.mjs"; + +function describe(plan, cached) { + console.log(`[verify] ${plan.kind === "full" ? "Full Vitest suite" : plan.kind === "broad" ? "Broader verification required" + : `${plan.selectedFiles.length}${plan.totalFiles === undefined ? "" : `/${plan.totalFiles}`} Vitest test files selected`}${cached ? "; reusing a passing local result" : ""}`); + if (plan.reason) console.log(`[verify] ${plan.reason}`); + if (plan.retainedFiles) console.log(`[verify] ${plan.retainedFiles} test files retained conservatively`); + if (plan.requiredCI.length) console.log(`[verify] Required CI coverage: ${plan.requiredCI.join(", ")}`); + console.log("[verify] Local Vitest coverage does not replace required CI, typecheck/build, browser, release, or platform-specific checks."); +} + +export async function verify({ root = process.cwd(), base, full = false, explain = false, noCache = false, env = localEnvironment() } = {}) { + root = realpathSync(root); + const before = snapshot(root, env), diff = comparison(root, base), key = requestKey(before, diff, full); + const canReuse = !noCache && before.cacheable && mayReuseAttempt(root, key); + const cached = canReuse ? readPass(root, key) : null; + console.log(`[verify] Base ${diff.base.slice(0, 12)}; ${diff.changed.length} changed paths, including local edits`); + if (cached) { + if (snapshot(root, env).hash !== before.hash) throw new Error("Inputs changed during cache lookup. Run verification again."); + describe(cached.plan, true); + console.log(`[verify] Passed result from ${cached.completedAt}; input fingerprint ${before.hash.slice(0, 12)}`); + if (explain) console.log(JSON.stringify({ ...diff, inputFingerprint: before.hash, cacheable: before.cacheable, plan: cached.plan }, null, 2)); + return 0; + } + if (!explain) { + invalidatePass(root, key); + writeRecord(root, `attempt-${key}`, { version: protocolVersion, key, status: "running", plan: null }); + } + const plan = (canReuse ? readPlan(root, key) : null) + ?? await planVerification({ root, env, diff, full }); + if (plan.manifest) plan.manifest.head = diff.head; + if (snapshot(root, env).hash !== before.hash) throw new Error("Inputs changed during selection. Run verification again."); + describe(plan, false); + if (explain) { + console.log(JSON.stringify({ ...diff, inputFingerprint: before.hash, cacheable: before.cacheable, plan }, null, 2)); + return plan.kind === "broad" ? 2 : 0; + } + if (plan.kind === "broad") { + console.log("[verify] Run focused checks and use required CI for broad coverage, or explicitly run pnpm verify --full."); + writeRecord(root, `attempt-${key}`, { version: protocolVersion, key, status: "broad", plan }); + return 2; + } + if (before.cacheable && !noCache && plan.kind === "selected") { + writeRecord(root, `plan-${key}`, { version: protocolVersion, key, status: "planned", plan }); + } + invalidatePass(root, key); + const startedAt = new Date().toISOString(); + let code = 0; + for (const job of plan.jobs) { + const jobKey = createHash("sha256").update(`${key}\0${job.id}`).digest("hex"); + if (canReuse && job.cacheable && readPass(root, jobKey)) { + console.log(`[verify] Reusing passed ${job.id}`); + continue; + } + invalidatePass(root, jobKey); + code = await runPlan(root, env, { ...plan, jobs: [job] }); + if (snapshot(root, env).hash !== before.hash) { + console.error("[verify] Inputs changed during verification. The result cannot be reused."); + code ||= 1; + } + if (code !== 0) break; + if (!noCache && before.cacheable && job.cacheable) writeRecord(root, jobKey, { + version: protocolVersion, key: jobKey, status: "passed", plan, jobId: job.id, completedAt: new Date().toISOString(), + }); + } + const after = snapshot(root, env); + if (after.hash !== before.hash) { + console.error("[verify] Inputs changed during verification. The result cannot be reused."); + code ||= 1; + } + const record = { version: protocolVersion, key, status: code === 0 ? "passed" : "failed", startedAt, + completedAt: new Date().toISOString(), head: diff.head, inputFingerprint: before.hash, plan, exitCode: code }; + writeRecord(root, "last-run", record); + writeRecord(root, `attempt-${key}`, record); + if (code === 0 && !noCache && before.cacheable && after.cacheable && plan.jobs.every(job => job.cacheable)) writeRecord(root, key, record); + console.log(`[verify] ${code === 0 ? "Local Vitest verification passed" : "Verification failed or interrupted"}${code === 0 && !before.cacheable ? "; external inputs disable pass reuse" : ""}`); + return code; +} + +async function main() { + const { values } = parseArgs({ options: { base: { type: "string" }, full: { type: "boolean" }, + explain: { type: "boolean" }, "no-cache": { type: "boolean" }, help: { type: "boolean" } } }); + if (values.help) { + console.log("Usage: pnpm verify [--base REF] [--explain] [--full] [--no-cache]\nRuns affected local Vitest checks. Exit 2 means broader coverage is required; use CI or --full."); + return; + } + process.exitCode = await verify({ base: values.base, full: values.full, explain: values.explain, noCache: values["no-cache"] }); +} + +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(error => { console.error(`[verify] ${error.message}`); process.exitCode = 1; }); +} diff --git a/scripts/local-verify/inputs.mjs b/scripts/local-verify/inputs.mjs new file mode 100644 index 00000000000..27474cce0fd --- /dev/null +++ b/scripts/local-verify/inputs.mjs @@ -0,0 +1,96 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync, realpathSync, readlinkSync } from "node:fs"; +import path from "node:path"; +import { sourceFiles } from "../ci/execution-plan.mjs"; +import { productInputs } from "../../.github/scripts/ci-inputs.mjs"; +import { installedInputs } from "./installed-inputs.mjs"; + +export const protocolVersion = 1; + +export function git(root, ...args) { + return execFileSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"] }); +} + +export function localEnvironment(env = process.env) { + const result = { ...env }; + delete result.GITHUB_WORKFLOW; + delete result.PAPERCLIP_CI_TEST_SELECTION; + delete result.PAPERCLIP_CI_SELECTION_HASH; + return result; +} + +export function comparison(root, requestedBase) { + let base; + for (const ref of requestedBase ? [requestedBase] : ["origin/master", "master"]) { + try { + const commit = git(root, "rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`).trim(); + base = git(root, "merge-base", "HEAD", commit).trim(); + break; + } catch {} + } + if (!base) throw new Error("Cannot resolve a base. Pass --base ."); + const changed = [...new Set([ + ...git(root, "diff", "--name-only", "--no-renames", "-z", base, "--").split("\0"), + ...git(root, "diff", "--cached", "--name-only", "--no-renames", "-z", base, "--").split("\0"), + ...git(root, "ls-files", "--others", "--exclude-standard", "-z").split("\0"), + ].filter(Boolean))].sort(); + return { base, head: git(root, "rev-parse", "HEAD").trim(), changed }; +} + +function version(root, env, command, args) { + try { return execFileSync(command, args, { cwd: root, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); } + catch { return "unavailable"; } +} + +export function snapshot(root, env = localEnvironment()) { + root = realpathSync(root); + const files = sourceFiles(root); + const hash = createHash("sha256"); + const append = record => hash.update(`${JSON.stringify(record)}\n`); + const digest = file => createHash("sha256").update(readFileSync(file)).digest("hex"); + let cacheable = true; + const addFile = file => { + const absolute = path.join(root, file); + try { + const stat = lstatSync(absolute); + if (stat.isSymbolicLink()) { + const target = realpathSync(absolute); + if (!target.startsWith(`${root}${path.sep}`)) cacheable = false; + const targetStat = lstatSync(target), regular = targetStat.isFile(); + if (!regular && !targetStat.isDirectory()) cacheable = false; + append(["link", file, stat.mode, readlinkSync(absolute), target, regular ? digest(target) : "directory"]); + } else if (stat.isFile()) append(["file", file, stat.mode, digest(absolute)]); + else { cacheable = false; append(["special", file, stat.mode]); } + } catch (error) { + if (error.code !== "ENOENT") throw error; + append(["missing", file]); + } + }; + productInputs(files).forEach(addFile); + const directories = new Set(["", ...files.filter(file => path.basename(file) === "package.json").map(path.dirname)]); + for (const directory of [...directories].sort()) { + for (const name of [".env", ".env.local", ".env.test", ".env.test.local"]) addFile(path.join(directory, name)); + } + installedInputs(root, [...directories].sort(), append); + for (const file of ["node_modules/.modules.yaml", "node_modules/.pnpm/lock.yaml"]) addFile(file); + const environment = Object.fromEntries(Object.keys(env).sort().map(name => [name, env[name]])); + if (Object.entries(env).some(([name, value]) => value?.trim() && ( + name === "DATABASE_URL" || name === "PAPERCLIP_TEST_POSTGRES_ADMIN_URL" + || /^PAPERCLIP_.*_TEST_DATABASE_URL$/.test(name) + || /^(?:PAPERCLIP|TEST)_.*(?:LIVE|E2E_LIVE)$/.test(name) && /^(?:1|true)$/i.test(value) + ))) cacheable = false; + const context = { version: protocolVersion, root, node: process.version, executable: realpathSync(process.execPath), + platform: process.platform, arch: process.arch, environment, + pnpm: version(root, env, "pnpm", ["--version"]), + rust: version(root, env, "rustc", ["--version", "--verbose"]), + cargo: version(root, env, "cargo", ["--version"]) }; + append(["context", context]); + return { hash: hash.digest("hex"), cacheable }; +} + +export function requestKey(inputs, diff, full) { + return createHash("sha256").update(JSON.stringify({ version: protocolVersion, inputs: inputs.hash, + base: diff.base, changed: diff.changed, full })).digest("hex"); +} diff --git a/scripts/local-verify/installed-inputs.mjs b/scripts/local-verify/installed-inputs.mjs new file mode 100644 index 00000000000..0410fafafb0 --- /dev/null +++ b/scripts/local-verify/installed-inputs.mjs @@ -0,0 +1,59 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, renameSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +const caches = new Map(); + +export function installedInputs(root, directories, append) { + const cacheFile = path.join(root, ".turbo/local-verify/installed-digests.json"); + if (!caches.has(root)) { + let entries = []; + try { + const saved = JSON.parse(readFileSync(cacheFile, "utf8")); + if (saved.version === 1) entries = Object.entries(saved.digests).filter(([, hash]) => /^[a-f0-9]{64}$/.test(hash)); + } catch {} + caches.set(root, new Map(entries)); + } + const digests = caches.get(root), current = new Map(); + let dirty = false; + const visited = new Set(); + const workspaces = new Set(directories.map(directory => path.join(root, directory))); + const visit = absolute => { + if (visited.has(absolute)) return; + visited.add(absolute); + let stat; + try { stat = lstatSync(absolute, { bigint: true }); } + catch (error) { + if (error.code !== "ENOENT") throw error; + append(["installed-missing", absolute]); + return; + } + if (stat.isSymbolicLink()) { + const target = readlinkSync(absolute); + append(["installed-link", absolute, String(stat.mode), target]); + visit(path.resolve(path.dirname(absolute), target)); + } else if (stat.isDirectory()) { + if (workspaces.has(absolute)) { visit(path.join(absolute, "node_modules")); return; } + const names = readdirSync(absolute).sort().filter(name => path.basename(absolute) !== "node_modules" + || ![".vite", ".vite-temp", ".cache"].includes(name)); + append(["installed-directory", absolute, String(stat.mode), names]); + for (const name of names) visit(path.join(absolute, name)); + } else if (stat.isFile()) { + const identity = [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs, stat.mode].join(":"); + if (!digests.has(identity)) { + digests.set(identity, createHash("sha256").update(readFileSync(absolute)).digest("hex")); + dirty = true; + } + current.set(identity, digests.get(identity)); + append(["installed-file", absolute, String(stat.mode), digests.get(identity)]); + } else throw new Error(`Unsupported installed input: ${absolute}`); + }; + for (const directory of directories) visit(path.join(root, directory, "node_modules")); + if (dirty || current.size !== digests.size) { + mkdirSync(path.dirname(cacheFile), { recursive: true }); + const temporary = `${cacheFile}.${randomUUID()}.tmp`; + writeFileSync(temporary, JSON.stringify({ version: 1, digests: Object.fromEntries(current) }), { mode: 0o600 }); + renameSync(temporary, cacheFile); + } + caches.set(root, current); +} diff --git a/scripts/local-verify/plan.mjs b/scripts/local-verify/plan.mjs new file mode 100644 index 00000000000..a29085fc7ad --- /dev/null +++ b/scripts/local-verify/plan.mjs @@ -0,0 +1,85 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { planTasks } from "../../.github/scripts/ci-plan.mjs"; +import { productInputs } from "../../.github/scripts/ci-inputs.mjs"; +import { selectionEligibility } from "../ci/select-tests.mjs"; + +const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts"; + +function inventory(root, env, group) { + const args = group === "serialized" ? ["--mode", "serialized"] : ["--mode", "general", "--group", group]; + const shards = ["general-workspaces-b", "general-workspaces-c"].includes(group) ? [] + : ["--shard-index", "0", "--shard-count", "1"]; + return JSON.parse(execFileSync(process.execPath, [path.join(root, "scripts/run-vitest-stable.mjs"), ...args, + ...shards, "--dry-run"], { + cwd: root, env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, + })); +} + +export async function planVerification({ root, env, diff, full = false }) { + const config = JSON.parse(readFileSync(path.join(root, "turbo.json"), "utf8")); + const lanes = JSON.parse(readFileSync(path.join(root, ".github/ci-task-matrix.json"), "utf8")); + const coarse = planTasks({ config, lanes, paths: diff.changed }); + const requiredCI = diff.changed.length ? Object.entries(coarse.jobs).filter(([, job]) => job.run).map(([name]) => name) : []; + if (full) return { kind: "full", jobs: [{ id: "full", args: null, cacheable: false }], requiredCI, selectedFiles: null, reason: "Full Vitest suite explicitly requested; opaque integration coverage runs fresh" }; + if (!productInputs(diff.changed).length) return { kind: "selected", jobs: [], requiredCI, selectedFiles: [], reason: "No changed product inputs" }; + const reason = coarse.forceReason ?? selectionEligibility(coarse, root); + if (reason) return { kind: "broad", reason, jobs: [], requiredCI }; + try { + const groups = [...new Set(["general_tests", "verify_serialized_server"].flatMap(job => + coarse.jobs[job]?.run ? coarse.jobs[job].matrix.include.map(entry => entry.group ?? "serialized") : []))]; + const inventories = new Map(groups.filter(group => group !== "general-chat").map(group => [group, inventory(root, env, group)])); + const projects = [...new Set([...inventories.values()].flatMap(value => value.workspaceProjects ?? ["@paperclipai/server"]))]; + const { selectProjects } = await import("../ci/select-projects.mjs"); + const graphs = await selectProjects({ root, projects, changed: productInputs(diff.changed), concurrency: 1, env }); + if (graphs.some(graph => !graph.isolated)) throw new Error("Non-isolated projects require broad verification"); + const covered = new Set(graphs.flatMap(graph => graph.coveredChanges)); + const uncovered = productInputs(diff.changed).filter(file => !covered.has(file)); + if (uncovered.length) throw new Error(`Changes outside resolved test graphs: ${uncovered.join(", ")}`); + const byProject = new Map(graphs.map(graph => [graph.project, graph])); + const manifest = { version: 1, head: diff.head, lanes: {} }, jobs = [], selectedFiles = new Set(); + for (const group of groups) { + if (group === "general-chat") { + if (byProject.get("@paperclipai/server")?.selected.includes(chatSuite)) { + selectedFiles.add(chatSuite); + for (let index = 0; index < 6; index++) jobs.push({ id: `chat-${index + 1}`, cacheable: false, args: ["--mode", "general", "--group", group, + "--shard-index", String(index), "--shard-count", "6", "--serial-chat"] }); + } + continue; + } + const value = inventories.get(group); + const owners = value.workspaceProjects + ? Object.fromEntries(value.workspaceProjects.map(project => [project, byProject.get(project).inventory])) + : { "@paperclipai/server": group === "serialized" ? value.selectedSerializedSuites : value.selectedGeneralServerSuites }; + const reusable = {}, fresh = {}; + for (const [project, files] of Object.entries(owners)) { + const graph = byProject.get(project), all = new Set(graph.inventory), affected = new Set(graph.selected); + if (!Array.isArray(files) || files.some(file => !all.has(file))) throw new Error(`Runner/graph inventory mismatch: ${group}`); + const matches = files.filter(file => affected.has(file)); + const opaque = new Set(graph.alwaysRun); + const pure = matches.filter(file => !opaque.has(file)), retained = matches.filter(file => opaque.has(file)); + if (pure.length) reusable[project] = pure; + if (retained.length) fresh[project] = retained; + matches.forEach(file => selectedFiles.add(file)); + } + const partitions = [reusable, fresh].filter(files => Object.keys(files).length); + if (!partitions.length) continue; + manifest.lanes[group] = { shards: partitions }; + for (const [index, files] of partitions.entries()) jobs.push({ id: `${group}-${files === reusable ? "reusable" : "fresh"}`, + cacheable: files === reusable, selected: true, args: ["--mode", group === "serialized" ? "serialized" : "general", + ...(group === "serialized" ? [] : ["--group", group]), "--shard-index", String(index), "--shard-count", String(partitions.length)] }); + } + const expected = new Set(graphs.flatMap(graph => graph.selected)); + if (expected.size !== selectedFiles.size || [...expected].some(file => !selectedFiles.has(file))) { + throw new Error("Selected tests are not completely covered by local runner ownership"); + } + const { validateSelection } = await import("../ci/test-selection-manifest.mjs"); + validateSelection(manifest); + return { kind: "selected", jobs, manifest, requiredCI, selectedFiles: [...selectedFiles].sort(), + totalFiles: graphs.reduce((count, graph) => count + graph.inventory.length, 0), + retainedFiles: graphs.reduce((count, graph) => count + graph.alwaysRun.length, 0) }; + } catch (error) { + return { kind: "broad", reason: `Conservative selection unavailable: ${error.message}`, jobs: [], requiredCI }; + } +} diff --git a/scripts/local-verify/run.mjs b/scripts/local-verify/run.mjs new file mode 100644 index 00000000000..b3d72c025c0 --- /dev/null +++ b/scripts/local-verify/run.mjs @@ -0,0 +1,86 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { createHash, randomUUID } from "node:crypto"; +import { selectionHash } from "../ci/test-selection-manifest.mjs"; +import { protocolVersion } from "./inputs.mjs"; + +export const cacheDirectory = root => path.join(root, ".turbo/local-verify"); +const planHash = plan => createHash("sha256").update(JSON.stringify(plan)).digest("hex"); + +export function readPass(root, key) { + try { + const result = JSON.parse(readFileSync(path.join(cacheDirectory(root), `${key}.json`), "utf8")); + return result.version === protocolVersion && result.key === key && result.status === "passed" && result.planHash === planHash(result.plan) + && ["selected", "full"].includes(result.plan?.kind) && Array.isArray(result.plan.jobs) ? result : null; + } catch { return null; } +} + +export function readPlan(root, key) { + try { + const result = JSON.parse(readFileSync(path.join(cacheDirectory(root), `plan-${key}.json`), "utf8")); + return result.version === protocolVersion && result.key === key && result.status === "planned" && result.planHash === planHash(result.plan) + && result.plan?.kind === "selected" && Array.isArray(result.plan.jobs) ? result.plan : null; + } catch { return null; } +} + +export function mayReuseAttempt(root, key) { + try { + const result = JSON.parse(readFileSync(path.join(cacheDirectory(root), `attempt-${key}.json`), "utf8")); + return result.version === protocolVersion && result.key === key && result.status === "passed"; + } catch (error) { return error.code === "ENOENT"; } +} + +export function writeRecord(root, name, record) { + const directory = cacheDirectory(root); + mkdirSync(directory, { recursive: true }); + const temporary = path.join(directory, `${name}.${randomUUID()}.tmp`); + writeFileSync(temporary, JSON.stringify({ ...record, planHash: planHash(record.plan) }, null, 2), { mode: 0o600 }); + renameSync(temporary, path.join(directory, `${name}.json`)); +} + +export const invalidatePass = (root, key) => rmSync(path.join(cacheDirectory(root), `${key}.json`), { force: true }); + +async function execute(command, args, root, env, label) { + return new Promise(resolve => { + const grouped = process.platform !== "win32"; + const child = spawn(command, args, { cwd: root, env, stdio: "inherit", detached: grouped }); + let stopped; + let finished = false; + const interrupt = signal => { + stopped = signal; + try { + if (grouped && child.pid) process.kill(-child.pid, signal); + else child.kill(signal); + } catch (error) { if (error.code !== "ESRCH") throw error; } + }; + const onInterrupt = () => interrupt("SIGINT"), onTerminate = () => interrupt("SIGTERM"); + process.on("SIGINT", onInterrupt); process.on("SIGTERM", onTerminate); + const finish = code => { + if (finished) return; + finished = true; + process.off("SIGINT", onInterrupt); process.off("SIGTERM", onTerminate); + resolve(stopped ? stopped === "SIGINT" ? 130 : 143 : code); + }; + child.once("error", error => { console.error(error.message); finish(1); }); + child.once("close", code => finish(code ?? 1)); + console.log(`[verify] Running ${label}`); + }); +} + +export async function runPlan(root, env, plan) { + const directory = cacheDirectory(root); + mkdirSync(directory, { recursive: true }); + const manifestFile = path.join(directory, `selection-${randomUUID()}.json`); + if (plan.manifest) writeFileSync(manifestFile, JSON.stringify(plan.manifest)); + try { + for (const job of plan.jobs) { + const code = job.args === null ? await execute("pnpm", ["test:run"], root, env, job.id) + : await execute(process.execPath, [path.join(root, "scripts/run-vitest-stable.mjs"), ...job.args], root, + job.selected ? { ...env, PAPERCLIP_CI_TEST_SELECTION: manifestFile, + PAPERCLIP_CI_SELECTION_HASH: selectionHash(plan.manifest) } : env, job.id); + if (code !== 0) return code; + } + return 0; + } finally { rmSync(manifestFile, { force: true }); } +} diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index f5dde1a91eb..dd1ec2da91f 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -267,6 +267,7 @@ function parseCliOptions(argv) { let shardCount = null; let group = null; let dryRun = false; + let serialChat = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -312,6 +313,11 @@ function parseCliOptions(argv) { continue; } + if (arg === "--serial-chat") { + serialChat = true; + continue; + } + if (arg === "--group") { group = readOptionValue(argv, index, arg); index += 1; @@ -337,10 +343,12 @@ function parseCliOptions(argv) { const shardAllowed = mode === serializedModeName || (mode === generalModeName && - ([...generalServerGroups, generalChatGroupName, generalWorkspacesAGroupName].includes(group))); + ([...generalServerGroups, generalChatGroupName, generalWorkspacesAGroupName].includes(group) + || Boolean(process.env.PAPERCLIP_CI_TEST_SELECTION) + && [generalWorkspacesBGroupName, generalWorkspacesCGroupName].includes(group))); if (!shardAllowed && shardIndex !== null) { fail( - "--shard-index/--shard-count are only valid with serialized mode or a shardable general server/chat/workspaces-a group.", + "--shard-index/--shard-count require serialized mode, a shardable general server/chat/workspaces-a group, or a selected workspace group.", ); } @@ -352,6 +360,10 @@ function parseCliOptions(argv) { fail(`Unknown group "${group}". Expected one of: ${allowedGeneralGroupNames.join(", ")}.`); } + if (serialChat && (mode !== generalModeName || group !== generalChatGroupName)) { + fail("--serial-chat is only valid with --mode general --group general-chat."); + } + if (shardIndex !== null) { if (shardIndex >= shardCount) { fail(`--shard-index must be less than --shard-count. Received ${shardIndex} of ${shardCount}.`); @@ -374,6 +386,7 @@ function parseCliOptions(argv) { shardCount, group, dryRun, + serialChat, }; } @@ -493,7 +506,7 @@ async function runProjectGroup(projects, groupName, shardIndex = null, shardCoun } function chatShardWorkers(index, count) { - const workers = count > 1 && !hasSharedTestDatabase ? 2 : 1; + const workers = count > 1 && !hasSharedTestDatabase && !options.serialChat ? 2 : 1; return Array.from({ length: workers }, (_, offset) => ({ index: (index ?? 0) * workers + offset, count: (count ?? 1) * workers,