diff --git a/package.json b/package.json index 3c0097e99e8..95a615e405d 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "test:slack-connector": "node server/src/services/connectors/slack/evals/run-regressions.mjs", "test:e2e:browser-context": "playwright test --config tests/e2e/playwright-company-context.config.ts", "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", - "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts && node --test tests/runner-e2e/native-completion-checks.test.mjs tests/runner-e2e/native-completion-git-source.test.mjs tests/runner-e2e/native-completion-source-contract.test.mjs", + "test:e2e:runner:unit": "node tests/runner-e2e/run-unit.mjs", "test:e2e:runner:stock-harness": "node tests/runner-e2e/stock-harness-checks.mjs", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 61867bbda68..be25c94edab 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -624,6 +624,16 @@ pnpm test:e2e:runner:unit pnpm test:e2e:runner:typecheck ``` +The unit command removes ambient `NODE_PATH` and `NODE_OPTIONS` when it starts +the actual Vitest and Node test processes. It invokes Vitest's JavaScript entry +directly so pnpm's generated bin shim cannot inject `NODE_PATH` again. Installed +CLI guards still reject ambient injection and repository-only resolution. Test +file filters and Vitest options can follow `pnpm test:e2e:runner:unit`; the Node +prerequisite tests still run after Vitest succeeds. Missing dependencies or an +unavailable/terminated test process produce a `launcher/environment failure` +diagnostic and exit 2. Assertion failures retain the test runner's exit status +and output; assess installed-product failures from those assertions separately. + The OpenRouter ranking snapshot is tracked in `openrouter-models.json`; nightly runs never mutate it. Refresh it deliberately, review the source/capture/hash diff, and rerun credential-free checks: diff --git a/tests/runner-e2e/connection-support.test.ts b/tests/runner-e2e/connection-support.test.ts index bc46b3064b7..77738d80444 100644 --- a/tests/runner-e2e/connection-support.test.ts +++ b/tests/runner-e2e/connection-support.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { chmod, mkdtemp, writeFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { AGENT_ADAPTER_TYPES } from "../../packages/shared/src/constants.js"; @@ -179,6 +179,7 @@ describe("target and selected credential boundaries", () => { expect(await resolveConnectionSecret(config, undefined, {})).toBeUndefined(); await expect(resolveConnectionSecret(config, "XAI_API_KEY", {})).rejects.toThrow("missing_XAI_API_KEY"); await writeFile(path.join(root, "public"), "OPENAI_API_KEY=private", { mode: 0o644 }); + await chmod(path.join(root, "public"), 0o644); await expect(resolveConnectionSecret(parseConnectionConfig({ secretFile: path.join(root, "public") }), "OPENAI_API_KEY", {})).rejects.toThrow(/owner_only/); } finally { await rm(root, { recursive: true, force: true }); } }); diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index 44cd1ee82b1..b503b6b6d76 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -1,5 +1,5 @@ import { matchCopilotFixtureCommand } from "./copilot-protection-evidence.js"; -import { writeFileSync, unlinkSync } from "node:fs"; +import { writeFileSync, unlinkSync, utimesSync } from "node:fs"; import { spawn } from "node:child_process"; import { readFile, mkdtemp, rm, writeFile, unlink, rename, mkdir, symlink } from "node:fs/promises"; import { join } from "node:path"; @@ -92,6 +92,7 @@ describe("Copilot Product protection integration", () => { const fixture = await createDeniedTargetFixture(root, "denied"); try { writeFileSync(fixture.targetPath, "x"); unlinkSync(fixture.targetPath); + utimesSync(fixture.directory, 0, 0); const receipt = fixture.watcher.finish(); expect(receipt).toMatchObject({ complete: false, targetMutationCount: 0 }); expect(receipt.reasons).toContain("coverage-gap-parent-version-changed"); diff --git a/tests/runner-e2e/fixtures/native-instruction-sources.json.gz b/tests/runner-e2e/fixtures/native-instruction-sources.json.gz new file mode 100644 index 00000000000..abe6782fcfb Binary files /dev/null and b/tests/runner-e2e/fixtures/native-instruction-sources.json.gz differ diff --git a/tests/runner-e2e/fixtures/native-instruction-sources.md b/tests/runner-e2e/fixtures/native-instruction-sources.md new file mode 100644 index 00000000000..806bd95e42a --- /dev/null +++ b/tests/runner-e2e/fixtures/native-instruction-sources.md @@ -0,0 +1,28 @@ +# Historical native instruction source fixture + +`native-instruction-sources.json.gz` retains the UTF-8 source files used by the +historical instruction comparison. Unit validation can read these sources in a +shallow checkout without network access or Git history. The test reads the +contents as data. It does not execute the retained source code. + +The JSON maps each variant to its `sourceSha` and a `files` object. Each key in +`files` is a repository-relative path. Its value is the exact source text from +that commit. + +| Variant | Source commit | +| --- | --- | +| `baseline` | `2a8a99e4a5f69aa803b3f10b982f583e75a87042` | +| `opencodeFeedbackResponses` | `a386a599983519eb1d399f8b770bfccdb2a74762` | + +The paths and SHA-256 checks remain in `NATIVE_INSTRUCTION_VARIANTS` in +`../native-instruction-consolidation.ts`. The test checks both complete variants +against those hashes and still rejects changed or mixed source files. The +separate ancestry tests still use real disposable Git repositories. + +To regenerate the fixture, read each declared path with `git show SHA:path` +from the commits above. Store each result as UTF-8 text in the JSON object. +Serialize with sorted keys, no extra whitespace, and unescaped Unicode. +Compress with gzip and set its modification time to zero. Do not substitute +current source files or update the accepted hashes to match a fixture. + +Fixture SHA-256: `99ecbfdfceb5f9328cb937740bbe06e9fe797d19b4d9b07182a1a0976f2e5825`. diff --git a/tests/runner-e2e/native-instruction-consolidation.test.ts b/tests/runner-e2e/native-instruction-consolidation.test.ts index 4096905dba6..2cf461598eb 100644 --- a/tests/runner-e2e/native-instruction-consolidation.test.ts +++ b/tests/runner-e2e/native-instruction-consolidation.test.ts @@ -2,13 +2,13 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { gunzipSync } from "node:zlib"; import { describe, expect, it } from "vitest"; import { runnerSuites, buildRunnerMatrix } from "./catalog.js"; import { gradeNativeDefault, NATIVE_MASTER_DEFAULT_SHA256 } from "./native-completion-defaults.js"; import { nativeCompletionTasks } from "./native-completion-cases.js"; -import { nativeCompletionPreflightEnvironment } from "./native-completion-admission.js"; import { assertNativeInstructionLineage, assertNativeInstructionSelection, nativeInstructionVariant, validateNativeInstructionMeasurement, NATIVE_INSTRUCTION_BASE_SHA, NATIVE_INSTRUCTION_DEFAULT_SHA256, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_VARIANTS } from "./native-instruction-consolidation.js"; describe("native instruction comparison admission", () => { @@ -68,18 +68,17 @@ describe("native instruction comparison admission", () => { }); it("accepts each complete source variant and rejects a mixed variant", () => { const files = Object.keys(NATIVE_INSTRUCTION_VARIANTS.baseline); - const historicalFile = (sha: string, file: string) => execFileSync("git", ["--no-replace-objects", "show", `${sha}:${file}`], { - cwd: resolve(import.meta.dirname, "../.."), env: nativeCompletionPreflightEnvironment(process.env), timeout: 30_000, - }); - const baseline = new Map(files.map(file => [file, historicalFile(NATIVE_INSTRUCTION_BASE_SHA, file)])); + const sources = JSON.parse(gunzipSync(readFileSync(new URL("./fixtures/native-instruction-sources.json.gz", import.meta.url))).toString()) as + Record }>; + expect(sources.baseline!.sourceSha).toBe(NATIVE_INSTRUCTION_BASE_SHA); + expect(sources.opencodeFeedbackResponses!.sourceSha).toBe("a386a599983519eb1d399f8b770bfccdb2a74762"); + for (const source of Object.values(sources)) expect(Object.keys(source.files).sort()).toEqual([...files].sort()); + const baseline = new Map(files.map(file => [file, Buffer.from(sources.baseline!.files[file]!)])); expect(nativeInstructionVariant(file => baseline.get(file)!)).toBe("baseline"); - // This historical admission intentionally rejects later production changes. - // Exercise the retained qualified tree rather than requiring HEAD to remain - // one of this completed comparison's frozen variants forever. - const candidate = new Map(files.map(file => [file, historicalFile("16ef1a5744b23bb043a8b286e6288921da1de98f", file)])); + const candidate = new Map(files.map(file => [file, Buffer.from(sources.opencodeFeedbackResponses!.files[file]!)])); const currentVariant = nativeInstructionVariant(file => candidate.get(file)!); - expect(["baseline", "candidate", "corrected", "feedback", "opencodeFeedback", "opencodeFeedbackSettlement", "opencodeFeedbackResponses"]).toContain(currentVariant); - candidate.set(files[0]!, currentVariant === "candidate" ? baseline.get(files[0]!)! : Buffer.from("unknown source")); + expect(currentVariant).toBe("opencodeFeedbackResponses"); + candidate.set(files[0]!, baseline.get(files[0]!)!); expect(() => nativeInstructionVariant(file => candidate.get(file)!)).toThrow("Mixed or unknown"); baseline.set(files[0]!, Buffer.from("unknown source")); expect(() => nativeInstructionVariant(file => baseline.get(file)!)).toThrow("Mixed or unknown"); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs new file mode 100644 index 00000000000..d2a1f01848a --- /dev/null +++ b/tests/runner-e2e/run-unit.mjs @@ -0,0 +1,35 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const vitest = path.join(root, "node_modules/vitest/vitest.mjs"); +const prefix = "[runner-e2e:unit]"; +const args = process.argv.slice(2); +if (args[0] === "--") args.shift(); + +if (!existsSync(vitest)) { + console.error(`${prefix} launcher/environment failure: Vitest is not installed. Run pnpm install before validation.`); + process.exit(2); +} + +const env = { ...process.env }; +delete env.NODE_PATH; +delete env.NODE_OPTIONS; + +for (const [name, command] of [ + ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], + ["Node tests", ["--test", "tests/runner-e2e/native-completion-checks.test.mjs", + "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"]], +]) { + const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); + if (result.error || result.signal) { + console.error(`${prefix} launcher/environment failure: ${name} ${result.error?.message ?? `terminated by ${result.signal}`}`); + process.exit(2); + } + if (result.status !== 0) { + console.error(`${prefix} ${name} failed (exit ${result.status}); see test runner output above.`); + process.exit(result.status ?? 1); + } +} diff --git a/tests/runner-e2e/unit-launcher.test.mjs b/tests/runner-e2e/unit-launcher.test.mjs new file mode 100644 index 00000000000..b6672c06158 --- /dev/null +++ b/tests/runner-e2e/unit-launcher.test.mjs @@ -0,0 +1,60 @@ +import { spawnSync } from "node:child_process"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); + +it.skipIf(process.env.PAPERCLIP_UNIT_BOUNDARY_PROBE === "1")("runs the documented pnpm validation with clean test processes after parent and shim injection", () => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "runner unit boundary-"))); + try { + const preload = path.join(directory, "preload.cjs"); + const foreignModules = path.join(directory, "foreign-modules"); + const log = path.join(directory, "preloads.ndjson"); + const report = path.join(directory, "vitest.json"); + writeFileSync(preload, `require("node:fs").appendFileSync(${JSON.stringify(log)}, JSON.stringify({ + argv: process.argv, execArgv: process.execArgv, nodePath: process.env.NODE_PATH + }) + "\\n");`); + const result = spawnSync("pnpm", ["test:e2e:runner:unit", "installed-cli.test.ts", + "--maxWorkers=1", "--reporter=json", `--outputFile=${report}`], { + cwd: root, + encoding: "utf8", + timeout: 90_000, + env: { ...process.env, PAPERCLIP_UNIT_BOUNDARY_PROBE: "1", + NODE_PATH: foreignModules, NODE_OPTIONS: `--require=${JSON.stringify(preload)}` }, + }); + expect(result.error, `${result.stdout}\n${result.stderr}`).toBeUndefined(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const tests = JSON.parse(readFileSync(report, "utf8")); + expect(tests.numPassedTests).toBeGreaterThan(0); + expect(tests.numFailedTests).toBe(0); + expect(tests.testResults).toHaveLength(1); + expect(tests.testResults[0].name).toMatch(/installed-cli\.test\.ts$/); + expect(result.stdout).toContain("native completion prerequisite admits an executed passing boundary"); + const preloads = readFileSync(log, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(preloads.some(record => record.argv.some(arg => /pnpm(?:\.c?js)?$/.test(arg)))).toBe(true); + expect(preloads.some(record => record.argv[1]?.endsWith("/run-unit.mjs") + && record.nodePath?.split(path.delimiter).includes(foreignModules))).toBe(true); + expect(preloads.every(record => !/vitest|tinypool|\.test\.(?:ts|mjs)$/.test(record.argv[1] ?? "") + && !record.execArgv.includes("--test"))).toBe(true); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}, 120_000); + +it("reports a missing test runner as a launcher/environment failure before any product assertions", () => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "runner-unit-uninstalled-"))); + try { + const launcher = path.join(directory, "tests/runner-e2e/run-unit.mjs"); + mkdirSync(path.dirname(launcher), { recursive: true }); + copyFileSync(path.join(root, "tests/runner-e2e/run-unit.mjs"), launcher); + const result = spawnSync(process.execPath, [launcher], { encoding: "utf8", timeout: 5000 }); + expect(result.status).toBe(2); + expect(result.stderr).toContain("launcher/environment failure: Vitest is not installed"); + expect(result.stdout).toBe(""); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +});