Tighten ingestion performance and repeatability acceptance

scripts/benchmark-native-event-ingestion.ts

@@ -15,6 +15,7 @@ import { installDatabaseWorkSignals } from "../packages/db/src/work-signals.js";
 import type { AppendHeartbeatRunEventInput, AppendHeartbeatRunEventResult } from "../server/src/services/heartbeat-run-events.js";
 import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js";
 import { nativeSha256 } from "../server/src/services/native-runtime/canonical.js";
+import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
 
 // This measures the real database owner, not browser delivery or a provider.
 // The assembled application/browser acceptance is a separate required layer.
@@ -45,6 +46,9 @@ if (arg("validate-report")) {
   if (process.argv.includes("--require-acceptance")) {
     assertServiceAcceptance(recorded);
   }
+  if (process.argv.includes("--require-repeatability")) {
+    assert.equal(evaluateServiceRepeatability(recorded).passed, true, "A/A repeatability gate failed; retain evidence and diagnose before candidate measurements");
+  }
   console.log(JSON.stringify({ valid: true, phase: recorded.phase, baselineRevision: recorded.baselineRevision,
     candidateRevision: recorded.candidateRevision, scope: recorded.scope }));
   process.exit(0);
@@ -52,6 +56,7 @@ if (arg("validate-report")) {
 const quick = process.argv.includes("--quick");
 const phase = arg("phase") ?? "compare";
 assert.ok(["aa", "compare", "batch"].includes(phase), "--phase must be aa, compare, or batch");
+if (process.argv.includes("--require-repeatability")) assert.equal(phase, "aa", "Repeatability requires the A/A phase");
 const baselineRef = arg("baseline-ref");
 assert.ok(baselineRef, "Pass a pinned --baseline-ref=<commit>");
 const baselineRevision = git(["rev-parse", "--verify", `${baselineRef}^{commit}`]).trim();
@@ -107,6 +112,7 @@ const report: Record<string, unknown> = {
   environment: { node: process.version, platform: process.platform, architecture: process.arch, cpus: availableParallelism(), loadBefore },
   results: [], incomplete: true, cleanupComplete: false,
 };
+let calibratedNoise: any;
 
 async function save() {
   if (output) await fs.writeFile(output, JSON.stringify(report, null, 2));
@@ -216,13 +222,24 @@ try {
   report.effectiveCandidateRevision = phase === "aa" ? baselineRevision : candidateRevision;
   if (phase === "batch") assert.equal(typeof candidate.appendHeartbeatRunEvents, "function", "Batch phase requires appendHeartbeatRunEvents");
   report.candidateSourceSha256 = createHash("sha256").update(await fs.readFile(path.join(root, sourcePath))).digest("hex");
-  const proofFiles = [...new Set([sourcePath, "scripts/benchmark-native-event-ingestion.ts",
+  const proofFiles = [...new Set([sourcePath, "scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs",
     ...git(["diff", "--name-only", baselineRevision, "--", "server/src/services", "packages/db/src/schema"]).trim().split("\n"),
     ...git(["ls-files", "--others", "--exclude-standard", "--", "server/src/services", "packages/db/src/schema"]).trim().split("\n"),
-  ])].filter(file => file.endsWith(".ts") && !file.endsWith(".test.ts"));
+  ])].filter(file => /\.(?:ts|mjs)$/.test(file) && !file.endsWith(".test.ts"));
   report.candidateSourceHashes = Object.fromEntries(await Promise.all(proofFiles.map(async file => [file,
     createHash("sha256").update(await fs.readFile(path.join(root, file))).digest("hex")])));
   report.candidateDirtyDiff = git(["diff", "HEAD", "--", sourcePath, "server/src/services/heartbeat", "packages/db/src/schema"]);
+  if (arg("noise-report")) {
+    calibratedNoise = JSON.parse(await fs.readFile(arg("noise-report")!, "utf8"));
+    assert.equal(calibratedNoise.phase, "aa"); assert.equal(calibratedNoise.incomplete, false);
+    assert.equal(calibratedNoise.baselineRevision, baselineRevision); assert.equal(calibratedNoise.workloadSha256, workloadSha256);
+    for (const file of ["scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs"]) {
+      assert.equal(calibratedNoise.candidateSourceHashes[file], (report.candidateSourceHashes as Record<string, string>)[file],
+        "Measurement protocol or acceptance gates changed after A/A calibration; rerun calibration");
+    }
+    assert.equal(evaluateServiceRepeatability(calibratedNoise).passed, true,
+      "A/A repeatability must pass before candidate measurements start");
+  }
   if (!externalUrl) database = await startEmbeddedPostgresTestDatabase("native-event-ingestion-benchmark-");
   if (externalUrl) await applyPendingMigrations(externalUrl);
   db = createDb(externalUrl ?? database!.connectionString, { maxConnections: 1 });
@@ -413,20 +430,18 @@ try {
   report.incomplete = false;
   report.acceptance = phase === "aa" ? "Baseline repeatability only; this does not establish an improvement" : "Compare against the frozen same-workload A/A report; throughput improvement needs at least four of five paired wins beyond baseline noise. Browser, control and full-app gates remain separate.";
   if (arg("noise-report")) {
-    const noise = JSON.parse(await fs.readFile(arg("noise-report")!, "utf8"));
-    assert.equal(noise.phase, "aa"); assert.equal(noise.incomplete, false);
-    assert.equal(noise.baselineRevision, baselineRevision); assert.equal(noise.workloadSha256, workloadSha256);
-    assert.equal(noise.candidateSourceHashes["scripts/benchmark-native-event-ingestion.ts"],
-      (report.candidateSourceHashes as Record<string, string>)["scripts/benchmark-native-event-ingestion.ts"],
-      "Measurement protocol changed after A/A calibration; rerun calibration rather than reuse its noise envelope");
+    const noise = calibratedNoise;
     report.noiseReport = arg("noise-report");
     report.comparison = (report.results as any[]).map(result => {
       const control = noise.results.find((row: any) => row.history === result.history && row.scenario === result.scenario);
       assert.ok(control);
-      const improvement = 1 - result.candidate.medianMs / result.baseline.medianMs;
-      const improvedBeyondNoise = result.pairedCandidateWins >= Math.ceil(pairs * 0.8) && improvement > control.aaNoiseEnvelopeRelative;
+      const improvement = (result.baseline.medianMs - result.candidate.medianMs) / result.baseline.medianMs;
+      const paired = noiseExceedingPairedWins(result.baseline.samples.map((sample: Sample, index: number) =>
+        (sample.elapsedMs - result.candidate.samples[index].elapsedMs) / sample.elapsedMs), control.aaNoiseEnvelopeRelative);
+      const improvedBeyondNoise = paired.pairedWinRequirementPassed && improvement > control.aaNoiseEnvelopeRelative;
       const noRegressionBeyondNoise = -improvement <= control.aaNoiseEnvelopeRelative;
       return { history: result.history, scenario: result.scenario, pairedCandidateWins: result.pairedCandidateWins,
+        noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
         latencyGate: result.latencyGate,
         improvementRelative: improvement, noiseEnvelopeRelative: control.aaNoiseEnvelopeRelative,
         improvedBeyondNoise, noRegressionBeyondNoise,
@@ -458,5 +473,12 @@ try {
   report.cleanupComplete = true;
   await save();
 }
+if (phase === "aa") {
+  report.repeatability = evaluateServiceRepeatability(report);
+  await save();
+}
 console.log(JSON.stringify(report, null, 2));
 if (process.argv.includes("--require-acceptance")) assertServiceAcceptance(report);
+if (process.argv.includes("--require-repeatability")) {
+  assert.equal(evaluateServiceRepeatability(report).passed, true, "A/A repeatability gate failed; retain evidence and diagnose before candidate measurements");
+}

scripts/compare-native-ingestion-browser.ts

@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
 import { createHash } from "node:crypto";
 import { readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
 
 // Each manifest entry points at native-ingestion-evidence JSON attachments
 // from two isolated compiled-app runs. Run the variants in alternating order;
@@ -22,6 +23,8 @@ const first = pairs[0];
 for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
   const report = pair[arm];
   assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance");
+  assert.notEqual(report.diagnosticOnly, true, "Diagnostic profiles cannot qualify unprofiled performance");
+  assert.notEqual(report.performanceAcceptanceEligible, false, "Profile-only evidence cannot qualify performance");
   assert.equal(report.serverArtifact, "compiled", "Performance acceptance requires the compiled server and shipped UI");
   assert.ok(report.serverEntrySha256 && report.measurementSha256);
   assert.equal(report.compiledBuildRevision, report.sourceRevision, "Compiled build stamp does not match the measured revision");
@@ -55,7 +58,7 @@ const measurements = names.flatMap(name => {
       const baseline = metric(pair.baseline.cases.find((item: any) => item.name === name), key);
       const candidate = metric(pair.candidate.cases.find((item: any) => item.name === name), key);
       assert.ok(Number.isFinite(baseline) && baseline > 0 && Number.isFinite(candidate) && candidate >= 0, `Invalid latency ${name}.${key}`);
-      return { baseline, candidate, improvementRelative: 1 - candidate / baseline };
+      return { baseline, candidate, improvementRelative: (baseline - candidate) / baseline };
     });
     return { name, metric: key, baselineMedianMs: median(samples.map(sample => sample.baseline)),
       candidateMedianMs: median(samples.map(sample => sample.candidate)),
@@ -69,6 +72,8 @@ const result: Record<string, any> = {
   scope: "Compiled-app deterministic-provider/browser acceptance; live-provider qualification remains separate",
   phase, expectation, pairCount: pairs.length, baselineRevision: first.baseline.sourceRevision,
   candidateRevision: first.candidate.sourceRevision, measurementSha256: first.baseline.measurementSha256,
+  gatesSha256: createHash("sha256").update(await readFile(new URL("./native-ingestion-performance-gates.mjs", import.meta.url))).digest("hex"),
+  comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
   manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements,
   primaryLoadedWorkload: "burst512", complete: true,
 };
@@ -77,6 +82,8 @@ if (phase === "compare") {
   const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8"));
   assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true);
   assert.equal(noise.baselineRevision, result.baselineRevision); assert.equal(noise.measurementSha256, result.measurementSha256);
+  assert.equal(noise.gatesSha256, result.gatesSha256, "Acceptance gates changed after A/A calibration");
+  assert.equal(noise.comparatorSha256, result.comparatorSha256, "Comparator changed after A/A calibration");
   assert.ok(noise.pairCount >= 5);
   result.gates = measurements.map(item => {
     const control = noise.measurements.find((entry: any) => entry.name === item.name && entry.metric === item.metric);
@@ -84,9 +91,11 @@ if (phase === "compare") {
     const improvementRequired = expectation === "improvement" && item.name === "burst512"
       && ["emissionToVisible.p95Ms", "intendedToVisible.p95Ms", "drainThroughTerminalMs"].includes(item.metric);
     const noRegression = -item.medianPairedImprovementRelative <= control.aaNoiseEnvelopeRelative;
-    const improvement = item.candidateWins >= Math.ceil(pairs.length * 0.8)
+    const paired = noiseExceedingPairedWins(item.samples.map(sample => sample.improvementRelative), control.aaNoiseEnvelopeRelative);
+    const improvement = paired.pairedWinRequirementPassed
       && item.medianPairedImprovementRelative > control.aaNoiseEnvelopeRelative;
     return { name: item.name, metric: item.metric, improvementRequired, noiseEnvelopeRelative: control.aaNoiseEnvelopeRelative,
+      noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
       passed: improvementRequired ? improvement : noRegression };
   });
   result.acceptancePassed = result.gates.every((entry: any) => entry.passed);

scripts/native-ingestion-performance-gates.mjs

@@ -0,0 +1,51 @@
+// Shared by the real owner-path harness and compiled-browser comparator.
+export const SERVICE_REPEATABILITY_LIMITS = Object.freeze({
+  maximumPairedRelativeDifference: 0.20,
+  maximumMedianArmDrift: 0.10,
+});
+
+const median = values => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
+
+export function noiseExceedingPairedWins(improvements, noiseEnvelopeRelative) {
+  if (!Number.isFinite(noiseEnvelopeRelative) || noiseEnvelopeRelative < 0
+    || improvements.length < 5 || !improvements.every(Number.isFinite)) {
+    throw new Error("Finite noise and at least five paired improvements are required");
+  }
+  const requiredPairedWins = Math.ceil(improvements.length * 0.8);
+  const count = improvements.filter(value => value > noiseEnvelopeRelative).length;
+  return { noiseExceedingPairedWins: count, requiredPairedWins,
+    pairedWinRequirementPassed: count >= requiredPairedWins };
+}
+
+export function evaluateServiceRepeatability(report) {
+  const failures = [];
+  if (report.phase !== "aa") failures.push("Only A/A calibrates repeatability");
+  if (report.incomplete !== false || report.cleanupComplete !== true) failures.push("Evidence and resource cleanup must be complete");
+  const workload = report.workload ?? {};
+  if (workload.quick !== false) failures.push("Smoke cannot qualify repeatability");
+  if (!Number.isInteger(workload.pairs) || workload.pairs < 5) failures.push("At least five alternating pairs are required");
+  if (workload.events !== 32 || workload.groups !== 16) failures.push("The frozen 16 groups of 32 events are required");
+  const histories = [1000, 10000, 100000], scenarios = ["ordered", "replay", "late-gap", "warm-suffix"];
+  if (!histories.every(value => workload.histories?.includes(value))
+    || !scenarios.every(value => workload.scenarios?.includes(value))) failures.push("The full fixed workload matrix is required");
+  const cases = [];
+  for (const history of histories) for (const scenario of scenarios) {
+    const rows = (report.results ?? []).filter(item => item.history === history && item.scenario === scenario);
+    const row = rows[0], baseline = row?.baseline?.samples, repeat = row?.candidate?.samples;
+    if (rows.length !== 1 || !Array.isArray(baseline) || !Array.isArray(repeat)
+      || baseline.length !== workload.pairs || repeat.length !== workload.pairs
+      || ![...baseline, ...repeat].every(item => Number.isFinite(item.elapsedMs) && item.elapsedMs > 0)) {
+      failures.push(`Missing or invalid paired samples: ${history}/${scenario}`); continue;
+    }
+    if (scenario === "late-gap") continue;
+    const maximumPairedRelativeDifference = Math.max(...baseline.map((sample, index) => Math.abs(repeat[index].elapsedMs - sample.elapsedMs) / sample.elapsedMs));
+    const baselineMedian = median(baseline.map(item => item.elapsedMs));
+    const medianArmDrift = Math.abs(median(repeat.map(item => item.elapsedMs)) - baselineMedian) / baselineMedian;
+    cases.push({ history, scenario, maximumPairedRelativeDifference, medianArmDrift,
+      passed: maximumPairedRelativeDifference <= SERVICE_REPEATABILITY_LIMITS.maximumPairedRelativeDifference
+        && medianArmDrift <= SERVICE_REPEATABILITY_LIMITS.maximumMedianArmDrift });
+  }
+  return { limits: SERVICE_REPEATABILITY_LIMITS, eligible: failures.length === 0,
+    eligibilityFailures: failures, cases, passed: failures.length === 0 && cases.length === 9 && cases.every(item => item.passed),
+    excluded: "Late-gap single-operation latency is structural-only; its complete paired evidence is still required" };
+}

scripts/native-ingestion-performance-gates.test.mjs

@@ -0,0 +1,127 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
+
+test("four pairs must individually improve beyond noise", () => {
+  const threeBeyondNoise = noiseExceedingPairedWins([0.30, 0.30, 0.30, 0.01, -0.01], 0.10);
+  assert.equal(threeBeyondNoise.noiseExceedingPairedWins, 3);
+  assert.equal(threeBeyondNoise.pairedWinRequirementPassed, false);
+  assert.equal(noiseExceedingPairedWins([0.30, 0.25, 0.20, 0.15, 0.01], 0.10).pairedWinRequirementPassed, true);
+  assert.equal(noiseExceedingPairedWins([0.30, 0.30, 0.30, 0.10, -0.01], 0.10).pairedWinRequirementPassed, false);
+});
+
+function repeatabilityReport() {
+  return { phase: "aa", incomplete: false, cleanupComplete: true,
+    workload: { quick: false, pairs: 5, events: 32, groups: 16, histories: [1000, 10000, 100000],
+      scenarios: ["ordered", "replay", "late-gap", "warm-suffix"] },
+    results: [1000, 10000, 100000].flatMap(history => ["ordered", "replay", "late-gap", "warm-suffix"].map(scenario => ({ history, scenario,
+      baseline: { samples: Array.from({ length: 5 }, () => ({ elapsedMs: 100 })) },
+      candidate: { samples: Array.from({ length: 5 }, () => ({ elapsedMs: 101 })) },
+    }))) };
+}
+
+test("service A/A enforces both preregistered limits on every timing case", () => {
+  assert.equal(evaluateServiceRepeatability(repeatabilityReport()).passed, true);
+  const variance = repeatabilityReport(); variance.results[11].candidate.samples[0].elapsedMs = 121;
+  assert.equal(evaluateServiceRepeatability(variance).passed, false);
+  const drift = repeatabilityReport(); drift.results[0].candidate.samples.forEach(value => { value.elapsedMs = 111; });
+  assert.equal(evaluateServiceRepeatability(drift).passed, false);
+  const boundary = repeatabilityReport(); boundary.results[0].candidate.samples.forEach(value => { value.elapsedMs = 110; });
+  boundary.results[0].candidate.samples[0].elapsedMs = 120;
+  assert.equal(evaluateServiceRepeatability(boundary).passed, true, "The stated inclusive 20%/10% boundaries qualify");
+  const late = repeatabilityReport(); late.results[2].candidate.samples[0].elapsedMs = 1000;
+  assert.equal(evaluateServiceRepeatability(late).passed, true, "Late-gap time stays structural-only");
+});
+
+test("smoke, partial, missing matrix, insufficient pairs and unfinished cleanup cannot qualify A/A", () => {
+  for (const change of [report => { report.workload.quick = true; }, report => { report.incomplete = true; },
+    report => { report.cleanupComplete = false; }, report => { report.workload.pairs = 4; },
+    report => { report.results.pop(); }, report => { report.phase = "compare"; }]) {
+    const report = repeatabilityReport(); change(report);
+    assert.equal(evaluateServiceRepeatability(report).passed, false);
+  }
+});
+
+test("owner report-validation CLI enforces repeatability without opening a database", async () => {
+  const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-owner-gates-"));
+  const root = path.resolve(import.meta.dirname, "..");
+  const report = repeatabilityReport();
+  report.candidateSourceHashes = Object.fromEntries(await Promise.all([
+    "scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs",
+  ].map(async file => [file, createHash("sha256").update(await readFile(path.join(root, file))).digest("hex")])));
+  const file = path.join(directory, "report.json");
+  const run = () => spawnSync(process.execPath, ["--import", "./server/node_modules/tsx/dist/loader.mjs",
+    "scripts/benchmark-native-event-ingestion.ts", `--validate-report=${file}`, "--require-repeatability"], { cwd: root, encoding: "utf8" });
+  try {
+    await writeFile(file, JSON.stringify(report));
+    const valid = run(); assert.equal(valid.status, 0, valid.stderr);
+    report.results[0].candidate.samples[0].elapsedMs = 150;
+    await writeFile(file, JSON.stringify(report));
+    const invalid = run(); assert.notEqual(invalid.status, 0); assert.match(invalid.stderr, /repeatability gate failed/);
+  } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+const workloads = [
+  { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0 },
+  { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0 },
+  { name: "burst128", runs: 10, events: 128, intervalMs: 0, prelude: 0 },
+  { name: "burst512", runs: 100, events: 512, intervalMs: 0, prelude: 0 },
+  { name: "existing-history", runs: 10, events: 128, intervalMs: 0, prelude: 1000 },
+  { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
+];
+function browserReport(revision, latency) {
+  return { complete: true, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true,
+    sourceRevision: revision, compiledBuildRevision: revision, serverArtifact: "compiled", serverEntrySha256: `${revision}-compiled`,
+    measurementSha256: "synthetic-fixed-fixture", compiledHotPathHashes: { append: "append", port: "port", runtime: "runtime" },
+    cases: workloads.map(workload => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true,
+      firstVisibleAfterEmissionMs: latency, emissionToVisible: { medianMs: latency, p95Ms: latency },
+      intendedToVisible: { p95Ms: latency }, drainThroughTerminalMs: latency,
+      stopThroughTerminalMs: latency, stopThroughAcknowledgementMs: latency,
+      durable: Array.from({ length: workload.runs }, (_, index) => ({ token: String(index), ordered: true,
+        uniqueSourceIdentities: workload.events, individualDeltas: workload.events })),
+    })) };
+}
+
+test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => {
+  const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-"));
+  const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
+  const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; };
+  async function manifest(prefix, improvements, sameRevision = false, mutate) {
+    const entries = [];
+    for (let index = 0; index < 5; index++) {
+      const baseline = browserReport("baseline", 1000), candidate = browserReport(sameRevision ? "baseline" : "candidate", 1000 * (1 - improvements[index]));
+      if (mutate && index === 0) mutate(candidate);
+      entries.push({ baselineFile: await save(`${prefix}-${index}-baseline.json`, baseline),
+        candidateFile: await save(`${prefix}-${index}-candidate.json`, candidate) });
+    }
+    return save(`${prefix}-pairs.json`, entries);
+  }
+  const run = args => spawnSync(process.execPath, [script.pathname, ...args], { encoding: "utf8" });
+  try {
+    const noise = path.join(directory, "noise.json");
+    const aa = run(["--phase=aa", `--pairs=${await manifest("aa", Array(5).fill(0.10), true)}`, `--output=${noise}`]);
+    assert.equal(aa.status, 0, aa.stderr);
+    const badOutput = path.join(directory, "three-wins.json");
+    const bad = run([`--pairs=${await manifest("three-wins", [0.30, 0.30, 0.30, 0.01, -0.01])}`, `--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]);
+    assert.notEqual(bad.status, 0);
+    const result = JSON.parse(await readFile(badOutput, "utf8"));
+    assert.equal(result.acceptancePassed, false);
+    assert.ok(result.gates.filter(gate => gate.improvementRequired).every(gate => gate.noiseExceedingPairedWins === 3));
+    const good = run([`--pairs=${await manifest("four-wins", [0.30, 0.25, 0.20, 0.15, 0.01])}`, `--noise-report=${noise}`, "--require-acceptance"]);
+    assert.equal(good.status, 0, good.stderr);
+    for (const [name, change, message] of [
+      ["smoke", report => { report.qualificationOnly = true; }, /Smoke cannot qualify/],
+      ["source", report => { report.serverArtifact = "source-qualification"; }, /compiled server/],
+      ["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/],
+      ["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/],
+    ]) {
+      const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30), false, change)}`, `--noise-report=${noise}`, "--require-acceptance"]);
+      assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message);
+    }
+  } finally { await rm(directory, { recursive: true, force: true }); }
+});