From 6205b24d541735b4ae2155b3779debeecbe88dd3 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 16 Nov 2026 16:19:59 -0800 Subject: [PATCH] fix(ci): review the instance settings cache reset in the server test setup (#21943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - CI selects only the server tests that a pull request can affect. The Vitest selection graph trusts a setup file only while its content matches a reviewed hash, and it follows every local import of that setup > - #21768 added a `beforeEach` to `server/src/__tests__/setup-supertest.ts` that resets the instance settings cache, through `../services/instance-settings-cache.js` > - The reviewed hash was not updated, and the cache module was not reviewed > - So server test selection fails with "Opaque setup dependency", and CI runs every server test instead. The qualification test also fails on every pull request that changes CI files > - This pull request reviews the new setup content and the cache reset > - The benefit is that server test selection works again, and pull requests that change CI files pass `qualify_test_selection` ## Linked Issues or Issue Description Refs #21768. **What happened?** `qualify_test_selection` ("Incremental CI qualification") fails on pull requests that change CI files, for example `perf/canary-git-numbering` and `fix/server-shard2-hang`: ``` ✖ real graph retains direct and indirect filesystem, process and opaque dependencies Error: Opaque setup dependency: .../server/src/__tests__/setup-supertest.ts ``` With only the hash updated, selection of the real server project stops at `Opaque setup package: drizzle-orm`, which `packages/db/src/work-signals.ts` imports through the cache module. **Expected behavior** The reviewed setup permits file selection again. A change to the setup, its helpers or the cache module still selects every server test. **Steps to reproduce** 1. On `master`, run `node --test scripts/ci/test-selection.integration.test.mjs`. 2. The real graph test fails with "Opaque setup dependency". ## What Changed - `scripts/ci/vitest-selection-graph.mjs`: - Pin the current `setup-supertest.ts` content. - Add `server/src/services/instance-settings-cache.ts` as a reviewed setup dependency. The setup calls only `invalidateInstanceSettingsCache`, which increments an in-memory generation. The module performs no IO when it loads. Its `drizzle-orm`, `drizzle-orm/pg-core` and `@paperclipai/db/work-signals` imports serve the read-through path, which the setup does not call. The review lists them as external, like `supertest/lib/test.js` for the setup itself. - `scripts/ci/test-selection.integration.test.mjs`: the fixture copies the cache module with the reviewed setup files. It also checks that a change to the cache module selects every test. ## Verification - `node --test scripts/ci/test-selection*.test.mjs scripts/ci/vitest-impact-report.integration.test.mjs` (the `qualify_test_selection` step): 42 passed. On `master`, the real graph case fails. - The "Qualify selection against the real Paperclip workspace" script from `ci-incremental-qualification.yml`, run locally with `NODE_ENV=test VITEST=true`: passes. - Real server project selection (`selectProjectTests` with the `@paperclipai/server` project): a change to `server/src/services/heartbeat-run-summary.ts` selects 2004 of 2188 tests. A change to `instance-settings-cache.ts` sets `setupChanged` and selects all 2188. ## Risks - A later edit to `instance-settings-cache.ts` or `setup-supertest.ts` disables the review again until the hash is renewed. Selection then fails closed and runs every server test, as it does today. - Changes to `packages/db/src/work-signals.ts` no longer select every server test through the setup. The setup's cache reset does not call it. Tests that import it directly are still selected. - CI scripts and tests only. No product code changes. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- scripts/ci/test-selection.integration.test.mjs | 6 +++++- scripts/ci/vitest-selection-graph.mjs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs index 38818dd5b34..64ce85b5814 100644 --- a/scripts/ci/test-selection.integration.test.mjs +++ b/scripts/ci/test-selection.integration.test.mjs @@ -88,7 +88,9 @@ test("real graph retains direct and indirect filesystem, process and opaque depe write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(process.env.INPUT);'); await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph"); mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true }); - for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) { + mkdirSync(path.join(directory, "server/src/services"), { recursive: true }); + for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts", + "server/src/services/instance-settings-cache.ts"]) { write(file, readFileSync(path.join(root, file), "utf8")); } write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";'); @@ -97,6 +99,8 @@ test("real graph retains direct and indirect filesystem, process and opaque depe const helper = "server/src/__tests__/helpers/isolated-test-home.ts"; const changedSetup = await select([helper]); assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed"); + const changedCache = await select(["server/src/services/instance-settings-cache.ts"]); + assert.deepEqual(changedCache.selected, changedCache.inventory, "the reviewed settings cache reset still invalidates all tests when changed"); write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`); await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects"); }); diff --git a/scripts/ci/vitest-selection-graph.mjs b/scripts/ci/vitest-selection-graph.mjs index 6e6949ea5b0..c288665119d 100644 --- a/scripts/ci/vitest-selection-graph.mjs +++ b/scripts/ci/vitest-selection-graph.mjs @@ -28,9 +28,13 @@ const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", " // following local imports even for these files; helper changes affect all tests. const reviewedSetup = new Map([ ["server/src/__tests__/setup-supertest.ts", { - hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb", + hash: "15a53631d5015910d4a3e9506158f9e4bd8269ac340afb52d94089da77fc3ced", external: ["supertest/lib/test.js"], }], + ["server/src/services/instance-settings-cache.ts", { + hash: "d5535ee52e7a8e2c776534a67912a30632ed79e6adf7a0a6929844244d387ff5", + external: ["drizzle-orm", "drizzle-orm/pg-core", "@paperclipai/db/work-signals"], + }], ["server/src/__tests__/helpers/isolated-test-home.ts", { hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b", external: [],