From aad61573e6c65eb0169bb86aa54e820258c85637 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 13:05:50 -0600 Subject: [PATCH] Tighten ingestion performance and repeatability acceptance --- .../2026-11-03-native-event-ingestion.md | 35 ++++- scripts/benchmark-native-event-ingestion.ts | 42 ++++-- scripts/compare-native-ingestion-browser.ts | 13 +- .../native-ingestion-performance-gates.mjs | 51 +++++++ ...ative-ingestion-performance-gates.test.mjs | 127 ++++++++++++++++++ tests/e2e/native-event-ingestion.journey.ts | 2 + 6 files changed, 255 insertions(+), 15 deletions(-) create mode 100644 scripts/native-ingestion-performance-gates.mjs create mode 100644 scripts/native-ingestion-performance-gates.test.mjs diff --git a/doc/plans/2026-11-03-native-event-ingestion.md b/doc/plans/2026-11-03-native-event-ingestion.md index 9b73ec22a0c..34727ef872d 100644 --- a/doc/plans/2026-11-03-native-event-ingestion.md +++ b/doc/plans/2026-11-03-native-event-ingestion.md @@ -81,8 +81,8 @@ has cursor projections is prepared with its own pinned rebuild helper. Both optimized arms must have current projections before steady-state timing. ```sh -EVENT_BASELINE_REF=2e37c73c05fe84aead1883b72fdf6b1653b58c2a -node --import ./server/node_modules/tsx/dist/loader.mjs scripts/benchmark-native-event-ingestion.ts --baseline-ref="$EVENT_BASELINE_REF" --phase=aa --output=/private/tmp/native-event-aa.json +EVENT_BASELINE_REF=83b5f7d486e70e484acd9a41bc957054f4e10175 +node --import ./server/node_modules/tsx/dist/loader.mjs scripts/benchmark-native-event-ingestion.ts --baseline-ref="$EVENT_BASELINE_REF" --phase=aa --output=/private/tmp/native-event-aa.json --require-repeatability node --import ./server/node_modules/tsx/dist/loader.mjs scripts/benchmark-native-event-ingestion.ts --baseline-ref="$EVENT_BASELINE_REF" --phase=compare --noise-report=/private/tmp/native-event-aa.json --output=/private/tmp/native-event-cursor.json --require-acceptance ``` @@ -94,6 +94,35 @@ sizes. Warm suffixes start above source sequence 1 and retain a contiguous curso of zero. Report the actual history before and after each sample; the size label is the seeded history, not an assertion that appended rows do not accumulate. +Fresh B0 is the exact unoptimized parent of B1, currently +`83b5f7d486e70e484acd9a41bc957054f4e10175`. The initially pinned +`2e37c73c05fe84aead1883b72fdf6b1653b58c2a` remains historical calibration/smoke +evidence. Aligning the control removes intervening unrelated recovery, GitHub, +CI and idle-history changes from application comparisons; it changes no +workload, durability, cache or acceptance contract. Pin the final B1/B2 commits +and matching compiled build stamps before fresh measurements. + +The lead approved explicit A/A repeatability limits before any revised timings: +every ordered/replay/warm-suffix case at each history size must have maximum +absolute paired relative latency difference **≤20%** and absolute median-arm +drift **≤10%**. All nine timing cases must pass; late-gap timing stays structural +only. `--require-repeatability` records these gates from raw samples and exits +unsuccessfully for failure, smoke, insufficient pairs, a missing matrix or +unfinished evidence/cleanup. It can also check an existing report with +`--validate-report= --require-repeatability`. A comparison validates this +calibration before starting candidate database work. Failed calibration stops +the sequence; preserve evidence and diagnose without weakening the limits. + +Improvement additionally requires at least four of five **individual paired +improvements to exceed the observed A/A noise**, plus median improvement beyond +that noise. Four merely positive changes do not suffice: with 10% noise, +`[30%, 30%, 30%, 1%, -1%]` fails. Shared gates and browser CLI rejection of +smoke/source/profile evidence are checked by: + +```sh +node --test scripts/native-ingestion-performance-gates.test.mjs +``` + Setup, projection rebuild, warmup, and independent verification stay outside the timed append interval. Post-sample reads verify durable count, global sequence, source identity, canonical hash, schema version and exact payload. SQL diagnostics @@ -232,7 +261,7 @@ node scripts/compare-native-ingestion-browser.ts --pairs=/private/tmp/browser-ab The preregistered primary loaded case is `burst512`: p95 emission/intended-to- visible latency and drainage through terminal settlement must improve beyond -A/A noise in at least four of five pairs. Other case metrics are nonregression +A/A noise in at least four of five individual pairs. Other case metrics are nonregression controls. Use `--expect=nonregression` for the cursor-only application control; the combined change still requires the loaded improvement. Comparison rejects smoke, source-server evidence, mismatched artifacts/protocols, missing background diff --git a/scripts/benchmark-native-event-ingestion.ts b/scripts/benchmark-native-event-ingestion.ts index b20eaee4bbd..bd05c8b3536 100644 --- a/scripts/benchmark-native-event-ingestion.ts +++ b/scripts/benchmark-native-event-ingestion.ts @@ -15,6 +15,7 @@ import { installDatabaseWorkSignals } from "../packages/db/src/work-signals.js"; import type { AppendHeartbeatRunEventInput, AppendHeartbeatRunEventResult } from "../server/src/services/heartbeat-run-events.js"; import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js"; import { nativeSha256 } from "../server/src/services/native-runtime/canonical.js"; +import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; // This measures the real database owner, not browser delivery or a provider. // The assembled application/browser acceptance is a separate required layer. @@ -45,6 +46,9 @@ if (arg("validate-report")) { if (process.argv.includes("--require-acceptance")) { assertServiceAcceptance(recorded); } + if (process.argv.includes("--require-repeatability")) { + assert.equal(evaluateServiceRepeatability(recorded).passed, true, "A/A repeatability gate failed; retain evidence and diagnose before candidate measurements"); + } console.log(JSON.stringify({ valid: true, phase: recorded.phase, baselineRevision: recorded.baselineRevision, candidateRevision: recorded.candidateRevision, scope: recorded.scope })); process.exit(0); @@ -52,6 +56,7 @@ if (arg("validate-report")) { const quick = process.argv.includes("--quick"); const phase = arg("phase") ?? "compare"; assert.ok(["aa", "compare", "batch"].includes(phase), "--phase must be aa, compare, or batch"); +if (process.argv.includes("--require-repeatability")) assert.equal(phase, "aa", "Repeatability requires the A/A phase"); const baselineRef = arg("baseline-ref"); assert.ok(baselineRef, "Pass a pinned --baseline-ref="); const baselineRevision = git(["rev-parse", "--verify", `${baselineRef}^{commit}`]).trim(); @@ -107,6 +112,7 @@ const report: Record = { environment: { node: process.version, platform: process.platform, architecture: process.arch, cpus: availableParallelism(), loadBefore }, results: [], incomplete: true, cleanupComplete: false, }; +let calibratedNoise: any; async function save() { if (output) await fs.writeFile(output, JSON.stringify(report, null, 2)); @@ -216,13 +222,24 @@ try { report.effectiveCandidateRevision = phase === "aa" ? baselineRevision : candidateRevision; if (phase === "batch") assert.equal(typeof candidate.appendHeartbeatRunEvents, "function", "Batch phase requires appendHeartbeatRunEvents"); report.candidateSourceSha256 = createHash("sha256").update(await fs.readFile(path.join(root, sourcePath))).digest("hex"); - const proofFiles = [...new Set([sourcePath, "scripts/benchmark-native-event-ingestion.ts", + const proofFiles = [...new Set([sourcePath, "scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs", ...git(["diff", "--name-only", baselineRevision, "--", "server/src/services", "packages/db/src/schema"]).trim().split("\n"), ...git(["ls-files", "--others", "--exclude-standard", "--", "server/src/services", "packages/db/src/schema"]).trim().split("\n"), - ])].filter(file => file.endsWith(".ts") && !file.endsWith(".test.ts")); + ])].filter(file => /\.(?:ts|mjs)$/.test(file) && !file.endsWith(".test.ts")); report.candidateSourceHashes = Object.fromEntries(await Promise.all(proofFiles.map(async file => [file, createHash("sha256").update(await fs.readFile(path.join(root, file))).digest("hex")]))); report.candidateDirtyDiff = git(["diff", "HEAD", "--", sourcePath, "server/src/services/heartbeat", "packages/db/src/schema"]); + if (arg("noise-report")) { + calibratedNoise = JSON.parse(await fs.readFile(arg("noise-report")!, "utf8")); + assert.equal(calibratedNoise.phase, "aa"); assert.equal(calibratedNoise.incomplete, false); + assert.equal(calibratedNoise.baselineRevision, baselineRevision); assert.equal(calibratedNoise.workloadSha256, workloadSha256); + for (const file of ["scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs"]) { + assert.equal(calibratedNoise.candidateSourceHashes[file], (report.candidateSourceHashes as Record)[file], + "Measurement protocol or acceptance gates changed after A/A calibration; rerun calibration"); + } + assert.equal(evaluateServiceRepeatability(calibratedNoise).passed, true, + "A/A repeatability must pass before candidate measurements start"); + } if (!externalUrl) database = await startEmbeddedPostgresTestDatabase("native-event-ingestion-benchmark-"); if (externalUrl) await applyPendingMigrations(externalUrl); db = createDb(externalUrl ?? database!.connectionString, { maxConnections: 1 }); @@ -413,20 +430,18 @@ try { report.incomplete = false; report.acceptance = phase === "aa" ? "Baseline repeatability only; this does not establish an improvement" : "Compare against the frozen same-workload A/A report; throughput improvement needs at least four of five paired wins beyond baseline noise. Browser, control and full-app gates remain separate."; if (arg("noise-report")) { - const noise = JSON.parse(await fs.readFile(arg("noise-report")!, "utf8")); - assert.equal(noise.phase, "aa"); assert.equal(noise.incomplete, false); - assert.equal(noise.baselineRevision, baselineRevision); assert.equal(noise.workloadSha256, workloadSha256); - assert.equal(noise.candidateSourceHashes["scripts/benchmark-native-event-ingestion.ts"], - (report.candidateSourceHashes as Record)["scripts/benchmark-native-event-ingestion.ts"], - "Measurement protocol changed after A/A calibration; rerun calibration rather than reuse its noise envelope"); + const noise = calibratedNoise; report.noiseReport = arg("noise-report"); report.comparison = (report.results as any[]).map(result => { const control = noise.results.find((row: any) => row.history === result.history && row.scenario === result.scenario); assert.ok(control); - const improvement = 1 - result.candidate.medianMs / result.baseline.medianMs; - const improvedBeyondNoise = result.pairedCandidateWins >= Math.ceil(pairs * 0.8) && improvement > control.aaNoiseEnvelopeRelative; + const improvement = (result.baseline.medianMs - result.candidate.medianMs) / result.baseline.medianMs; + const paired = noiseExceedingPairedWins(result.baseline.samples.map((sample: Sample, index: number) => + (sample.elapsedMs - result.candidate.samples[index].elapsedMs) / sample.elapsedMs), control.aaNoiseEnvelopeRelative); + const improvedBeyondNoise = paired.pairedWinRequirementPassed && improvement > control.aaNoiseEnvelopeRelative; const noRegressionBeyondNoise = -improvement <= control.aaNoiseEnvelopeRelative; return { history: result.history, scenario: result.scenario, pairedCandidateWins: result.pairedCandidateWins, + noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins, latencyGate: result.latencyGate, improvementRelative: improvement, noiseEnvelopeRelative: control.aaNoiseEnvelopeRelative, improvedBeyondNoise, noRegressionBeyondNoise, @@ -458,5 +473,12 @@ try { report.cleanupComplete = true; await save(); } +if (phase === "aa") { + report.repeatability = evaluateServiceRepeatability(report); + await save(); +} console.log(JSON.stringify(report, null, 2)); if (process.argv.includes("--require-acceptance")) assertServiceAcceptance(report); +if (process.argv.includes("--require-repeatability")) { + assert.equal(evaluateServiceRepeatability(report).passed, true, "A/A repeatability gate failed; retain evidence and diagnose before candidate measurements"); +} diff --git a/scripts/compare-native-ingestion-browser.ts b/scripts/compare-native-ingestion-browser.ts index fb15edfb1a3..2512188ee2a 100644 --- a/scripts/compare-native-ingestion-browser.ts +++ b/scripts/compare-native-ingestion-browser.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; +import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; // Each manifest entry points at native-ingestion-evidence JSON attachments // from two isolated compiled-app runs. Run the variants in alternating order; @@ -22,6 +23,8 @@ const first = pairs[0]; for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) { const report = pair[arm]; assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance"); + assert.notEqual(report.diagnosticOnly, true, "Diagnostic profiles cannot qualify unprofiled performance"); + assert.notEqual(report.performanceAcceptanceEligible, false, "Profile-only evidence cannot qualify performance"); assert.equal(report.serverArtifact, "compiled", "Performance acceptance requires the compiled server and shipped UI"); assert.ok(report.serverEntrySha256 && report.measurementSha256); assert.equal(report.compiledBuildRevision, report.sourceRevision, "Compiled build stamp does not match the measured revision"); @@ -55,7 +58,7 @@ const measurements = names.flatMap(name => { const baseline = metric(pair.baseline.cases.find((item: any) => item.name === name), key); const candidate = metric(pair.candidate.cases.find((item: any) => item.name === name), key); assert.ok(Number.isFinite(baseline) && baseline > 0 && Number.isFinite(candidate) && candidate >= 0, `Invalid latency ${name}.${key}`); - return { baseline, candidate, improvementRelative: 1 - candidate / baseline }; + return { baseline, candidate, improvementRelative: (baseline - candidate) / baseline }; }); return { name, metric: key, baselineMedianMs: median(samples.map(sample => sample.baseline)), candidateMedianMs: median(samples.map(sample => sample.candidate)), @@ -69,6 +72,8 @@ const result: Record = { scope: "Compiled-app deterministic-provider/browser acceptance; live-provider qualification remains separate", phase, expectation, pairCount: pairs.length, baselineRevision: first.baseline.sourceRevision, candidateRevision: first.candidate.sourceRevision, measurementSha256: first.baseline.measurementSha256, + gatesSha256: createHash("sha256").update(await readFile(new URL("./native-ingestion-performance-gates.mjs", import.meta.url))).digest("hex"), + comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"), manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements, primaryLoadedWorkload: "burst512", complete: true, }; @@ -77,6 +82,8 @@ if (phase === "compare") { const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8")); assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true); assert.equal(noise.baselineRevision, result.baselineRevision); assert.equal(noise.measurementSha256, result.measurementSha256); + assert.equal(noise.gatesSha256, result.gatesSha256, "Acceptance gates changed after A/A calibration"); + assert.equal(noise.comparatorSha256, result.comparatorSha256, "Comparator changed after A/A calibration"); assert.ok(noise.pairCount >= 5); result.gates = measurements.map(item => { const control = noise.measurements.find((entry: any) => entry.name === item.name && entry.metric === item.metric); @@ -84,9 +91,11 @@ if (phase === "compare") { const improvementRequired = expectation === "improvement" && item.name === "burst512" && ["emissionToVisible.p95Ms", "intendedToVisible.p95Ms", "drainThroughTerminalMs"].includes(item.metric); const noRegression = -item.medianPairedImprovementRelative <= control.aaNoiseEnvelopeRelative; - const improvement = item.candidateWins >= Math.ceil(pairs.length * 0.8) + const paired = noiseExceedingPairedWins(item.samples.map(sample => sample.improvementRelative), control.aaNoiseEnvelopeRelative); + const improvement = paired.pairedWinRequirementPassed && item.medianPairedImprovementRelative > control.aaNoiseEnvelopeRelative; return { name: item.name, metric: item.metric, improvementRequired, noiseEnvelopeRelative: control.aaNoiseEnvelopeRelative, + noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins, passed: improvementRequired ? improvement : noRegression }; }); result.acceptancePassed = result.gates.every((entry: any) => entry.passed); diff --git a/scripts/native-ingestion-performance-gates.mjs b/scripts/native-ingestion-performance-gates.mjs new file mode 100644 index 00000000000..745d1635d15 --- /dev/null +++ b/scripts/native-ingestion-performance-gates.mjs @@ -0,0 +1,51 @@ +// Shared by the real owner-path harness and compiled-browser comparator. +export const SERVICE_REPEATABILITY_LIMITS = Object.freeze({ + maximumPairedRelativeDifference: 0.20, + maximumMedianArmDrift: 0.10, +}); + +const median = values => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; + +export function noiseExceedingPairedWins(improvements, noiseEnvelopeRelative) { + if (!Number.isFinite(noiseEnvelopeRelative) || noiseEnvelopeRelative < 0 + || improvements.length < 5 || !improvements.every(Number.isFinite)) { + throw new Error("Finite noise and at least five paired improvements are required"); + } + const requiredPairedWins = Math.ceil(improvements.length * 0.8); + const count = improvements.filter(value => value > noiseEnvelopeRelative).length; + return { noiseExceedingPairedWins: count, requiredPairedWins, + pairedWinRequirementPassed: count >= requiredPairedWins }; +} + +export function evaluateServiceRepeatability(report) { + const failures = []; + if (report.phase !== "aa") failures.push("Only A/A calibrates repeatability"); + if (report.incomplete !== false || report.cleanupComplete !== true) failures.push("Evidence and resource cleanup must be complete"); + const workload = report.workload ?? {}; + if (workload.quick !== false) failures.push("Smoke cannot qualify repeatability"); + if (!Number.isInteger(workload.pairs) || workload.pairs < 5) failures.push("At least five alternating pairs are required"); + if (workload.events !== 32 || workload.groups !== 16) failures.push("The frozen 16 groups of 32 events are required"); + const histories = [1000, 10000, 100000], scenarios = ["ordered", "replay", "late-gap", "warm-suffix"]; + if (!histories.every(value => workload.histories?.includes(value)) + || !scenarios.every(value => workload.scenarios?.includes(value))) failures.push("The full fixed workload matrix is required"); + const cases = []; + for (const history of histories) for (const scenario of scenarios) { + const rows = (report.results ?? []).filter(item => item.history === history && item.scenario === scenario); + const row = rows[0], baseline = row?.baseline?.samples, repeat = row?.candidate?.samples; + if (rows.length !== 1 || !Array.isArray(baseline) || !Array.isArray(repeat) + || baseline.length !== workload.pairs || repeat.length !== workload.pairs + || ![...baseline, ...repeat].every(item => Number.isFinite(item.elapsedMs) && item.elapsedMs > 0)) { + failures.push(`Missing or invalid paired samples: ${history}/${scenario}`); continue; + } + if (scenario === "late-gap") continue; + const maximumPairedRelativeDifference = Math.max(...baseline.map((sample, index) => Math.abs(repeat[index].elapsedMs - sample.elapsedMs) / sample.elapsedMs)); + const baselineMedian = median(baseline.map(item => item.elapsedMs)); + const medianArmDrift = Math.abs(median(repeat.map(item => item.elapsedMs)) - baselineMedian) / baselineMedian; + cases.push({ history, scenario, maximumPairedRelativeDifference, medianArmDrift, + passed: maximumPairedRelativeDifference <= SERVICE_REPEATABILITY_LIMITS.maximumPairedRelativeDifference + && medianArmDrift <= SERVICE_REPEATABILITY_LIMITS.maximumMedianArmDrift }); + } + return { limits: SERVICE_REPEATABILITY_LIMITS, eligible: failures.length === 0, + eligibilityFailures: failures, cases, passed: failures.length === 0 && cases.length === 9 && cases.every(item => item.passed), + excluded: "Late-gap single-operation latency is structural-only; its complete paired evidence is still required" }; +} diff --git a/scripts/native-ingestion-performance-gates.test.mjs b/scripts/native-ingestion-performance-gates.test.mjs new file mode 100644 index 00000000000..134b649d006 --- /dev/null +++ b/scripts/native-ingestion-performance-gates.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; + +test("four pairs must individually improve beyond noise", () => { + const threeBeyondNoise = noiseExceedingPairedWins([0.30, 0.30, 0.30, 0.01, -0.01], 0.10); + assert.equal(threeBeyondNoise.noiseExceedingPairedWins, 3); + assert.equal(threeBeyondNoise.pairedWinRequirementPassed, false); + assert.equal(noiseExceedingPairedWins([0.30, 0.25, 0.20, 0.15, 0.01], 0.10).pairedWinRequirementPassed, true); + assert.equal(noiseExceedingPairedWins([0.30, 0.30, 0.30, 0.10, -0.01], 0.10).pairedWinRequirementPassed, false); +}); + +function repeatabilityReport() { + return { phase: "aa", incomplete: false, cleanupComplete: true, + workload: { quick: false, pairs: 5, events: 32, groups: 16, histories: [1000, 10000, 100000], + scenarios: ["ordered", "replay", "late-gap", "warm-suffix"] }, + results: [1000, 10000, 100000].flatMap(history => ["ordered", "replay", "late-gap", "warm-suffix"].map(scenario => ({ history, scenario, + baseline: { samples: Array.from({ length: 5 }, () => ({ elapsedMs: 100 })) }, + candidate: { samples: Array.from({ length: 5 }, () => ({ elapsedMs: 101 })) }, + }))) }; +} + +test("service A/A enforces both preregistered limits on every timing case", () => { + assert.equal(evaluateServiceRepeatability(repeatabilityReport()).passed, true); + const variance = repeatabilityReport(); variance.results[11].candidate.samples[0].elapsedMs = 121; + assert.equal(evaluateServiceRepeatability(variance).passed, false); + const drift = repeatabilityReport(); drift.results[0].candidate.samples.forEach(value => { value.elapsedMs = 111; }); + assert.equal(evaluateServiceRepeatability(drift).passed, false); + const boundary = repeatabilityReport(); boundary.results[0].candidate.samples.forEach(value => { value.elapsedMs = 110; }); + boundary.results[0].candidate.samples[0].elapsedMs = 120; + assert.equal(evaluateServiceRepeatability(boundary).passed, true, "The stated inclusive 20%/10% boundaries qualify"); + const late = repeatabilityReport(); late.results[2].candidate.samples[0].elapsedMs = 1000; + assert.equal(evaluateServiceRepeatability(late).passed, true, "Late-gap time stays structural-only"); +}); + +test("smoke, partial, missing matrix, insufficient pairs and unfinished cleanup cannot qualify A/A", () => { + for (const change of [report => { report.workload.quick = true; }, report => { report.incomplete = true; }, + report => { report.cleanupComplete = false; }, report => { report.workload.pairs = 4; }, + report => { report.results.pop(); }, report => { report.phase = "compare"; }]) { + const report = repeatabilityReport(); change(report); + assert.equal(evaluateServiceRepeatability(report).passed, false); + } +}); + +test("owner report-validation CLI enforces repeatability without opening a database", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-owner-gates-")); + const root = path.resolve(import.meta.dirname, ".."); + const report = repeatabilityReport(); + report.candidateSourceHashes = Object.fromEntries(await Promise.all([ + "scripts/benchmark-native-event-ingestion.ts", "scripts/native-ingestion-performance-gates.mjs", + ].map(async file => [file, createHash("sha256").update(await readFile(path.join(root, file))).digest("hex")]))); + const file = path.join(directory, "report.json"); + const run = () => spawnSync(process.execPath, ["--import", "./server/node_modules/tsx/dist/loader.mjs", + "scripts/benchmark-native-event-ingestion.ts", `--validate-report=${file}`, "--require-repeatability"], { cwd: root, encoding: "utf8" }); + try { + await writeFile(file, JSON.stringify(report)); + const valid = run(); assert.equal(valid.status, 0, valid.stderr); + report.results[0].candidate.samples[0].elapsedMs = 150; + await writeFile(file, JSON.stringify(report)); + const invalid = run(); assert.notEqual(invalid.status, 0); assert.match(invalid.stderr, /repeatability gate failed/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +const workloads = [ + { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0 }, + { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0 }, + { name: "burst128", runs: 10, events: 128, intervalMs: 0, prelude: 0 }, + { name: "burst512", runs: 100, events: 512, intervalMs: 0, prelude: 0 }, + { name: "existing-history", runs: 10, events: 128, intervalMs: 0, prelude: 1000 }, + { name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true }, +]; +function browserReport(revision, latency) { + return { complete: true, qualificationOnly: false, diagnosticOnly: false, performanceAcceptanceEligible: true, + sourceRevision: revision, compiledBuildRevision: revision, serverArtifact: "compiled", serverEntrySha256: `${revision}-compiled`, + measurementSha256: "synthetic-fixed-fixture", compiledHotPathHashes: { append: "append", port: "port", runtime: "runtime" }, + cases: workloads.map(workload => ({ ...workload, stop: Boolean(workload.stop), persistedAfterReload: true, + firstVisibleAfterEmissionMs: latency, emissionToVisible: { medianMs: latency, p95Ms: latency }, + intendedToVisible: { p95Ms: latency }, drainThroughTerminalMs: latency, + stopThroughTerminalMs: latency, stopThroughAcknowledgementMs: latency, + durable: Array.from({ length: workload.runs }, (_, index) => ({ token: String(index), ordered: true, + uniqueSourceIdentities: workload.events, individualDeltas: workload.events })), + })) }; +} + +test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-")); + const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url); + const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; }; + async function manifest(prefix, improvements, sameRevision = false, mutate) { + const entries = []; + for (let index = 0; index < 5; index++) { + const baseline = browserReport("baseline", 1000), candidate = browserReport(sameRevision ? "baseline" : "candidate", 1000 * (1 - improvements[index])); + if (mutate && index === 0) mutate(candidate); + entries.push({ baselineFile: await save(`${prefix}-${index}-baseline.json`, baseline), + candidateFile: await save(`${prefix}-${index}-candidate.json`, candidate) }); + } + return save(`${prefix}-pairs.json`, entries); + } + const run = args => spawnSync(process.execPath, [script.pathname, ...args], { encoding: "utf8" }); + try { + const noise = path.join(directory, "noise.json"); + const aa = run(["--phase=aa", `--pairs=${await manifest("aa", Array(5).fill(0.10), true)}`, `--output=${noise}`]); + assert.equal(aa.status, 0, aa.stderr); + const badOutput = path.join(directory, "three-wins.json"); + const bad = run([`--pairs=${await manifest("three-wins", [0.30, 0.30, 0.30, 0.01, -0.01])}`, `--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]); + assert.notEqual(bad.status, 0); + const result = JSON.parse(await readFile(badOutput, "utf8")); + assert.equal(result.acceptancePassed, false); + assert.ok(result.gates.filter(gate => gate.improvementRequired).every(gate => gate.noiseExceedingPairedWins === 3)); + const good = run([`--pairs=${await manifest("four-wins", [0.30, 0.25, 0.20, 0.15, 0.01])}`, `--noise-report=${noise}`, "--require-acceptance"]); + assert.equal(good.status, 0, good.stderr); + for (const [name, change, message] of [ + ["smoke", report => { report.qualificationOnly = true; }, /Smoke cannot qualify/], + ["source", report => { report.serverArtifact = "source-qualification"; }, /compiled server/], + ["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/], + ["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/], + ]) { + const rejected = run([`--pairs=${await manifest(name, Array(5).fill(0.30), false, change)}`, `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(rejected.status, 0); assert.match(rejected.stderr, message); + } + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/tests/e2e/native-event-ingestion.journey.ts b/tests/e2e/native-event-ingestion.journey.ts index e9c35cc3719..f7f88991b0c 100644 --- a/tests/e2e/native-event-ingestion.journey.ts +++ b/tests/e2e/native-event-ingestion.journey.ts @@ -85,6 +85,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const report: Record = { dependency: "Deterministic Codex app-server fixture; real application/runtime/port/PostgreSQL/live WebSocket/shipped UI", variant: process.env.PAPERCLIP_INGESTION_VARIANT ?? "unlabelled", qualificationOnly: smoke, + diagnosticOnly: Boolean(process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR), + performanceAcceptanceEligible: !process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR, startedAtMs: Date.now(), emittedClock: "Provider process epoch; original schedule retained even if stdout backpressures", observedClock: "Browser epoch after a DOM update and one paint opportunity",