From b578b95d3257af93d3f2696b0203bb4bcb0c6bcf Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 04:58:41 -0600 Subject: [PATCH] test: bind candidate comparisons to qualified native builds --- ...1-04-native-event-ingestion-application.md | 38 +++++++++--- scripts/compare-native-ingestion-browser.ts | 18 ++++-- ...ative-ingestion-performance-gates.test.mjs | 59 ++++++++++++++++++- tests/e2e/fixtures/native-ingestion-h5.md | 24 ++++++-- tests/e2e/fixtures/native-ingestion-h5.mjs | 4 +- 5 files changed, 122 insertions(+), 21 deletions(-) diff --git a/doc/benchmarks/2026-11-04-native-event-ingestion-application.md b/doc/benchmarks/2026-11-04-native-event-ingestion-application.md index a00a6819dcb..84238540c61 100644 --- a/doc/benchmarks/2026-11-04-native-event-ingestion-application.md +++ b/doc/benchmarks/2026-11-04-native-event-ingestion-application.md @@ -1,6 +1,6 @@ # Native ingestion application acceptance -Status at 2026-11-04 10:28 UTC: **application acceptance remains open**. Source +Status at 2026-11-04 10:58 UTC: **application acceptance remains open**. Source append measurements passed their registered gates. No production-application A/A calibration, paired browser improvement, or deployed improvement is claimed. Both delivery PRs remain drafts. @@ -32,16 +32,40 @@ The pinned upstream integration preserves all upstream changes. Cursor migration `0430_mute_may_parker.sql` is byte-identical to the previous cursor migration `0429_deep_prima.sql`; upstream's distinct 0429 and the combined generated schema remain intact. SQL/service attribution still requires the recorded unchanged -append implementation, index and dependency checks. New matched baseline source -is `7550ffa5d6447d2615bc89fc5313f4cbb958b615`; its normal build and inventories -are pending. Fresh main and companion B0/B1 calibration must precede further +append implementation, index and dependency checks. New matched inputs are: + +| Role | Immutable application revision | Four-root inventory files | +| --- | --- | ---: | +| B0 | `7550ffa5d6447d2615bc89fc5313f4cbb958b615` | 12,930 | +| B1 | `2cf1454406cbd582a11f04476713eceb746eada5` | 12,944 | +| Unoptimized B2 base | `0b616ccc90bf43cb26adac056a22ff01a84f982b` | 12,952 | + +All three normal independent locked production builds exited zero. Their actual +shared native SHA256 is +`512875bd051b7f7a97c93d2aa63648c675a8a991e745c79bc7481ad2d2f02c66`, +with Rust tree `11a7ac118a438159c252dc0df4779aa56fec80d6`. Reuse was limited +to the same fifteen recorded native inputs in a fresh target. The previous +4a83 target and builds stayed unchanged. Lead review independently verified all +40 build/source/runtime evidence files and each full canonical inventory. +Qualified receipt SHA256 is +`5a849da1bc307f4f1fd05c70a75813f6f78acc9941a54cc01e945652a5a79d6b`; +inventory-summary SHA256 is +`e29a9a1a2b9481ef79f47c6b135e1d207e19d1ffc00833aa28b1f9163a210c5d`. +These are build qualifications, with no application measurements. Fresh main +and companion B0/B1 calibration must precede further optimization. Workloads, deadlines, noise limits and acceptance gates stay fixed. Review also found a prospective comparison defect: the old comparator required one Runner binary across both arms while the planned Rust candidate requires a -new qualified binary. The new registration will validate each arm against its -own qualified build and require stable samples; A/A retains one shared build. -This correction is under test before the new freeze. Historical H5b/H5c/H5d +new qualified binary. The corrected comparator validates each arm against its +own qualified build and requires stable samples; A/A retains one shared build. +The actual CLI regression fails on the historical comparator and passes the +correction, with five native-provenance cases in a passing 44-case tool suite. +H5e is frozen at `73ea8516739e81fa7f37cd67651b4a2c73674014`; the separate +blocked-Stop V4 is `a89f321ab5654211cbff7be1ee15c110b8db4a5f`. They change +baseline input references and registration metadata; H5e also includes this +provenance correction. Qualification and timing execution remain unreleased. +Historical H5b/H5c/H5d sources and recorded qualifications stay immutable; their unexecuted prospective Rust-candidate comparisons are not accepted evidence. diff --git a/scripts/compare-native-ingestion-browser.ts b/scripts/compare-native-ingestion-browser.ts index f9e3f206586..249696cadf8 100644 --- a/scripts/compare-native-ingestion-browser.ts +++ b/scripts/compare-native-ingestion-browser.ts @@ -281,9 +281,15 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) } assert.equal(remote.provider.fixtureSha256, report.fixtureSha256, "Staged provider fixture changed"); assert.ok(remote.runner.metadata?.prpTransportModes?.includes("dial_wss"), "Remote daemon must support verified WSS"); - for (const [group, field] of [["runner", "sha256"], ["provider", "nodeSha256"], ["provider", "wrapperSha256"], ["provider", "fixtureSha256"]]) { - assert.equal(remote[group][field], first.baseline.remote[group][field], "Remote provider/daemon artifact changed across matched samples"); + for (const field of ["nodeSha256", "wrapperSha256", "fixtureSha256"]) { + assert.equal(remote.provider[field], first.baseline.remote.provider[field], "Remote provider artifact changed across matched samples"); } + // The candidate may change Runner code. Its serving hash is checked against + // its own sealed qualified receipt by validateArmInventory before this loop. + // Legacy comparisons without that receipt, and A/A, require one shared binary. + const runnerReference = h5 && phase !== "aa" ? first[arm] : first.baseline; + assert.equal(remote.runner.sha256, runnerReference.remote.runner.sha256, "Qualified Runner binary changed between samples of an arm"); + if (h5) assert.equal(remote.runner.sha256, report.armInventory.binarySha256, "Serving Runner differs from its arm's qualified build"); assert.equal(report.admissionProtocol, "two-at-a-time-provider-and-clock-readiness-before-common-epoch", "Use the qualified pre-measurement admission protocol"); assert.equal(report.setupActivationConcurrency, 2, "Keep the qualified setup activation concurrency of two"); assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval"); @@ -303,8 +309,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm"); assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples"); assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples"); - if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) - assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples"); + if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"]) + assert.equal(report.armInventory[field], first[phase === "aa" ? "baseline" : arm].armInventory[field], "Full qualified artifact inventory changed between samples"); assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs"); assert.match(report.harnessProvenance?.sourceRevision ?? "", /^[a-f0-9]{40}$/, "Missing frozen harness source revision"); assert.equal(report.harnessProvenance.sourceStatus, ""); @@ -455,7 +461,7 @@ if (h5 && phase === "compare" && candidatePolicy === "final-b2") { assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms"); assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database"); const inventory = await validateArmInventory(path.resolve(arg("original-h4e-inventory")!), { evidence: proof, owningPlaywright: owner }); - for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const) + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"] as const) assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory"); originalH4eConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision, retryObservations: proof.cases.map((item: any) => ({ name: item.name, @@ -629,7 +635,7 @@ if (phase === "compare") { if (h5) { assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256); assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required"); - for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"]) assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A"); } assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol"); diff --git a/scripts/native-ingestion-performance-gates.test.mjs b/scripts/native-ingestion-performance-gates.test.mjs index 6b818188106..0962c9cb851 100644 --- a/scripts/native-ingestion-performance-gates.test.mjs +++ b/scripts/native-ingestion-performance-gates.test.mjs @@ -189,6 +189,7 @@ function browserReport(revision, latency) { function h5Report(revision, loadedLatency) { const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000); + if (revision === "B2") report.remote.runner.sha256 = "8".repeat(64); Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true, actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture", @@ -255,7 +256,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: { manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } }); const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision, - compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40), + compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: (variant === "B2" ? "8" : "9").repeat(40), binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }], artifactInventorySummary: inventorySummaryFile }); inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile, @@ -292,6 +293,22 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires } return save(`${prefix}-pairs.json`, entries); } + async function requalifyNative(entry, arm, binarySha256, rustSourceTree) { + const report = JSON.parse(await readFile(entry[`${arm}File`], "utf8")); + report.remote.runner.sha256 = binarySha256; + await writeFile(entry[`${arm}File`], JSON.stringify(report)); + const owner = JSON.parse(await readFile(entry[`${arm}PlaywrightFile`], "utf8")); + owner.suites[0].tests[0].results[0].attachments[0].body = Buffer.from(JSON.stringify(report)).toString("base64"); + await writeFile(entry[`${arm}PlaywrightFile`], JSON.stringify(owner)); + const sidecar = JSON.parse(await readFile(entry[`${arm}InventoryFile`], "utf8")); + const receipt = JSON.parse(await readFile(sidecar.qualifiedReceiptFile, "utf8")); + receipt.binarySha256 = binarySha256; receipt.rustSourceTree = rustSourceTree; + receipt.nativeStageHashes[0].binarySha256 = binarySha256; + sidecar.qualifiedReceiptFile = await save(`${path.basename(entry[`${arm}File`])}-qualified-native.json`, receipt); + sidecar.qualifiedReceiptSha256 = hash(await readFile(sidecar.qualifiedReceiptFile)); + sidecar.playwrightReportSha256 = hash(await readFile(entry[`${arm}PlaywrightFile`])); + await writeFile(entry[`${arm}InventoryFile`], JSON.stringify(sidecar)); + } try { const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-")); proof.protocolVersion = H4E_PROTOCOL; @@ -403,6 +420,45 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires const goodPairs = await manifest("good", 1000); const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]); assert.equal(good.status, 0, good.stderr); + const goodReport = JSON.parse(good.stdout); + assert.equal(goodReport.artifactInventories.baseline.binarySha256, "d".repeat(64)); + assert.equal(goodReport.artifactInventories.candidate.binarySha256, "8".repeat(64)); + assert.notEqual(goodReport.artifactInventories.baseline.rustSourceTree, goodReport.artifactInventories.candidate.rustSourceTree); + await context.test("each arm accepts its distinct qualified Runner binary", () => { + assert.equal(goodReport.performancePassed, true); assert.equal(goodReport.acceptancePassed, true); + assert.equal(goodReport.originalH4eConfirmation.inventory.binarySha256, "8".repeat(64)); + }); + await context.test("a serving binary outside its qualified receipt is rejected", async () => { + const attempt = run(["--candidate-policy=final-b2", `--pairs=${await manifest("unqualified-native", 1000, { + mutateArm: (report, { arm, index }) => { if (arm === "candidate" && index === 2) report.remote.runner.sha256 = "7".repeat(64); }, + })}`, `--noise-report=${noise}`, "--require-performance"]); + assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Serving native binary differs from qualified build/); + }); + await context.test("a second qualified binary within one arm is rejected", async () => { + const file = await manifest("changed-qualified-native", 1000), entries = JSON.parse(await readFile(file, "utf8")); + await requalifyNative(entries[2], "candidate", "7".repeat(64), "7".repeat(40)); + const attempt = run(["--candidate-policy=final-b2", `--pairs=${file}`, `--noise-report=${noise}`, "--require-performance"]); + assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/); + }); + await context.test("A/A cannot compare two qualified Runner binaries", async () => { + const file = await manifest("aa-changed-qualified-native", 181000, { sameRevision: true }), entries = JSON.parse(await readFile(file, "utf8")); + await requalifyNative(entries[0], "candidate", "7".repeat(64), "7".repeat(40)); + const attempt = run(["--phase=aa", `--pairs=${file}`]); + assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/); + }); + await context.test("later original-deadline proof must use the candidate's qualified binary", async () => { + const entry = { candidateFile: await save("changed-proof-evidence.json", proof), + candidatePlaywrightFile: await save("changed-proof-playwright.json", JSON.parse(await readFile(originalProof, "utf8"))), + candidateInventoryFile: await save("changed-proof-inventory.json", JSON.parse(await readFile(originalInventory, "utf8"))) }; + await requalifyNative(entry, "candidate", "7".repeat(64), "7".repeat(40)); + const result = path.join(directory, "changed-proof-result.json"); + const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, + `--original-h4e-proof=${entry.candidatePlaywrightFile}`, `--original-h4e-inventory=${entry.candidateInventoryFile}`, `--output=${result}`, "--require-acceptance"]); + assert.notEqual(attempt.status, 0); + const rejected = JSON.parse(await readFile(result, "utf8")); + assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false); + assert.match(rejected.originalH4eConfirmation.reason, /Original confirmation used a different full qualified artifact inventory/); + }); const completionRegressionPairs = await manifest("verified-drain-regression", 1000, { mutateArm: (report, { arm }) => { if (arm !== "candidate") return; @@ -424,7 +480,6 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires assert.equal(regressionGate.improvementRequired, true); assert.equal(regressionGate.passed, false); assert(completionRegressionReport.gates.filter(gate => gate.name === "burst512" && ["emissionToVisible.p95Ms", "intendedToVisible.p95Ms", "drainThroughTerminalMs"].includes(gate.metric)).every(gate => gate.passed)); - const goodReport = JSON.parse(good.stdout); const rate = goodReport.verifiedCompletionRates.find(item => item.name === "burst512"); assert.equal(rate.samples[0].measuredDeltas, 51200); assert.equal(rate.samples[0].baselineDeltasPerSecond, 51200000 / 180010); diff --git a/tests/e2e/fixtures/native-ingestion-h5.md b/tests/e2e/fixtures/native-ingestion-h5.md index a503f1cdca8..9dada688137 100644 --- a/tests/e2e/fixtures/native-ingestion-h5.md +++ b/tests/e2e/fixtures/native-ingestion-h5.md @@ -1,4 +1,4 @@ -# H5c canonical-settlement observation registration (prospective; no execution release) +# H5e matched-CF observation registration (prospective; no execution release) H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5` (`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`). @@ -8,14 +8,30 @@ concurrent remote participant and waits for the product's canonical settlement, including supported same-run recovery, within the same terminal deadline. It does not optimize the product or turn earlier failures into passes. -The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654` -and B1 `07061a601cdc20603c8c7a46f9ddbad7ac5f5939`. Raw B2 -`ca7cba171cd5198ed569d75012edc858041cbc44` is the unoptimized batching base, +The aligned fresh application baselines are B0 `7550ffa5d6447d2615bc89fc5313f4cbb958b615` +and B1 `2cf1454406cbd582a11f04476713eceb746eada5`. Both normally include common +upstream `cf51936587963c26e3ddc7eb1365a07b2f8df5a6`. Raw B2 +`0b616ccc90bf43cb26adac056a22ff01a84f982b` is the unoptimized batching base, not a qualified future performance candidate. A later candidate needs its actual new immutable revision, normal build receipt and inventory; no candidate reference or matching binary is presumed before those artifacts exist. The historical a365/d8cd/914 application failures remain evidence of their recorded revisions. +This prospective registration inherits H5d `2df9897e1ce2ca90be29fa097d776c3766e217ad`. +Only baseline input pins and the reviewed comparator binary-provenance correction +change: each arm must use its qualified Runner binary consistently, while A/A uses +one shared qualified build. Provider fixture hashes remain matched across arms. +The workload, oracle, deadlines, precision and suitability thresholds stay fixed. +The H5 registration digest changes because it includes the baseline references. +The old 4a83 campaign was operationally aborted after upstream changed exercised +remote checkpoint and ordinary UI Stop paths; preserve its completed/partial raw +attempts, cleanup failures and later owned cleanup separately. None supplies H5e +noise or a timing arm. Qualify the new actual normal builds, then collect fresh +main and companion B0/B1 A/A before further product optimization. No new build +receipt, native binary hash, inventory or execution release exists in this source +preparation. Historical H5b/c/d helpers remain immutable; their original comparator +cannot compare a distinct Rust B2 binary and must not execute this new campaign. + Select H5 explicitly with `PAPERCLIP_INGESTION_OBSERVATION_MODE=h5c-observe-original-sla-independent-agents-settlement`. The new default H4e mode uses the same independent-agent geometry with the original diff --git a/tests/e2e/fixtures/native-ingestion-h5.mjs b/tests/e2e/fixtures/native-ingestion-h5.mjs index b2efeda1713..aa6ac3e6ce6 100644 --- a/tests/e2e/fixtures/native-ingestion-h5.mjs +++ b/tests/e2e/fixtures/native-ingestion-h5.mjs @@ -7,8 +7,8 @@ export const H4E_PROTOCOL = 'v4e-independent-agents-settlement-original-deadline export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant', fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50, poolReuse: 'only-across-sequential-cases' }); -export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654', - B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' }); +export const H5_BASELINE_REVISIONS = Object.freeze({ B0: '7550ffa5d6447d2615bc89fc5313f4cbb958b615', + B1: '2cf1454406cbd582a11f04476713eceb746eada5' }); export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000, streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }); export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });