test: freeze matched CF native ingestion inputs

This commit is contained in:
Dotta committed 2026-11-04 04:46:03 -06:00
1 parent 2df9897e1c
commit 73ea851673
4 files changed
+91 -14

No files matched your search

+12 -6
View File
@@ -281,9 +281,15 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
}
assert.equal(remote.provider.fixtureSha256, report.fixtureSha256, "Staged provider fixture changed");
assert.ok(remote.runner.metadata?.prpTransportModes?.includes("dial_wss"), "Remote daemon must support verified WSS");
for (const [group, field] of [["runner", "sha256"], ["provider", "nodeSha256"], ["provider", "wrapperSha256"], ["provider", "fixtureSha256"]]) {
assert.equal(remote[group][field], first.baseline.remote[group][field], "Remote provider/daemon artifact changed across matched samples");
for (const field of ["nodeSha256", "wrapperSha256", "fixtureSha256"]) {
assert.equal(remote.provider[field], first.baseline.remote.provider[field], "Remote provider artifact changed across matched samples");
}
// The candidate may change Runner code. Its serving hash is checked against
// its own sealed qualified receipt by validateArmInventory before this loop.
// Legacy comparisons without that receipt, and A/A, require one shared binary.
const runnerReference = h5 && phase !== "aa" ? first[arm] : first.baseline;
assert.equal(remote.runner.sha256, runnerReference.remote.runner.sha256, "Qualified Runner binary changed between samples of an arm");
if (h5) assert.equal(remote.runner.sha256, report.armInventory.binarySha256, "Serving Runner differs from its arm's qualified build");
assert.equal(report.admissionProtocol, "two-at-a-time-provider-and-clock-readiness-before-common-epoch", "Use the qualified pre-measurement admission protocol");
assert.equal(report.setupActivationConcurrency, 2, "Keep the qualified setup activation concurrency of two");
assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval");
@@ -303,8 +309,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm");
assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples");
assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples");
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples");
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"])
assert.equal(report.armInventory[field], first[phase === "aa" ? "baseline" : arm].armInventory[field], "Full qualified artifact inventory changed between samples");
assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs");
assert.match(report.harnessProvenance?.sourceRevision ?? "", /^[a-f0-9]{40}$/, "Missing frozen harness source revision");
assert.equal(report.harnessProvenance.sourceStatus, "");
@@ -455,7 +461,7 @@ if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
const inventory = await validateArmInventory(path.resolve(arg("original-h4e-inventory")!), { evidence: proof, owningPlaywright: owner });
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"] as const)
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
originalH4eConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision,
retryObservations: proof.cases.map((item: any) => ({ name: item.name,
@@ -629,7 +635,7 @@ if (phase === "compare") {
if (h5) {
assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256);
assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required");
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"])
assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A");
}
assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol");
@@ -189,6 +189,7 @@ function browserReport(revision, latency) {
function h5Report(revision, loadedLatency) {
const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000);
if (revision === "B2") report.remote.runner.sha256 = "8".repeat(64);
Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE,
workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true,
actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture",
@@ -255,7 +256,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: {
manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } });
const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision,
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40),
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: (variant === "B2" ? "8" : "9").repeat(40),
binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }],
artifactInventorySummary: inventorySummaryFile });
inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile,
@@ -292,6 +293,22 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
}
return save(`${prefix}-pairs.json`, entries);
}
async function requalifyNative(entry, arm, binarySha256, rustSourceTree) {
const report = JSON.parse(await readFile(entry[`${arm}File`], "utf8"));
report.remote.runner.sha256 = binarySha256;
await writeFile(entry[`${arm}File`], JSON.stringify(report));
const owner = JSON.parse(await readFile(entry[`${arm}PlaywrightFile`], "utf8"));
owner.suites[0].tests[0].results[0].attachments[0].body = Buffer.from(JSON.stringify(report)).toString("base64");
await writeFile(entry[`${arm}PlaywrightFile`], JSON.stringify(owner));
const sidecar = JSON.parse(await readFile(entry[`${arm}InventoryFile`], "utf8"));
const receipt = JSON.parse(await readFile(sidecar.qualifiedReceiptFile, "utf8"));
receipt.binarySha256 = binarySha256; receipt.rustSourceTree = rustSourceTree;
receipt.nativeStageHashes[0].binarySha256 = binarySha256;
sidecar.qualifiedReceiptFile = await save(`${path.basename(entry[`${arm}File`])}-qualified-native.json`, receipt);
sidecar.qualifiedReceiptSha256 = hash(await readFile(sidecar.qualifiedReceiptFile));
sidecar.playwrightReportSha256 = hash(await readFile(entry[`${arm}PlaywrightFile`]));
await writeFile(entry[`${arm}InventoryFile`], JSON.stringify(sidecar));
}
try {
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
proof.protocolVersion = H4E_PROTOCOL;
@@ -403,6 +420,45 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
const goodPairs = await manifest("good", 1000);
const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]);
assert.equal(good.status, 0, good.stderr);
const goodReport = JSON.parse(good.stdout);
assert.equal(goodReport.artifactInventories.baseline.binarySha256, "d".repeat(64));
assert.equal(goodReport.artifactInventories.candidate.binarySha256, "8".repeat(64));
assert.notEqual(goodReport.artifactInventories.baseline.rustSourceTree, goodReport.artifactInventories.candidate.rustSourceTree);
await context.test("each arm accepts its distinct qualified Runner binary", () => {
assert.equal(goodReport.performancePassed, true); assert.equal(goodReport.acceptancePassed, true);
assert.equal(goodReport.originalH4eConfirmation.inventory.binarySha256, "8".repeat(64));
});
await context.test("a serving binary outside its qualified receipt is rejected", async () => {
const attempt = run(["--candidate-policy=final-b2", `--pairs=${await manifest("unqualified-native", 1000, {
mutateArm: (report, { arm, index }) => { if (arm === "candidate" && index === 2) report.remote.runner.sha256 = "7".repeat(64); },
})}`, `--noise-report=${noise}`, "--require-performance"]);
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Serving native binary differs from qualified build/);
});
await context.test("a second qualified binary within one arm is rejected", async () => {
const file = await manifest("changed-qualified-native", 1000), entries = JSON.parse(await readFile(file, "utf8"));
await requalifyNative(entries[2], "candidate", "7".repeat(64), "7".repeat(40));
const attempt = run(["--candidate-policy=final-b2", `--pairs=${file}`, `--noise-report=${noise}`, "--require-performance"]);
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/);
});
await context.test("A/A cannot compare two qualified Runner binaries", async () => {
const file = await manifest("aa-changed-qualified-native", 181000, { sameRevision: true }), entries = JSON.parse(await readFile(file, "utf8"));
await requalifyNative(entries[0], "candidate", "7".repeat(64), "7".repeat(40));
const attempt = run(["--phase=aa", `--pairs=${file}`]);
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/);
});
await context.test("later original-deadline proof must use the candidate's qualified binary", async () => {
const entry = { candidateFile: await save("changed-proof-evidence.json", proof),
candidatePlaywrightFile: await save("changed-proof-playwright.json", JSON.parse(await readFile(originalProof, "utf8"))),
candidateInventoryFile: await save("changed-proof-inventory.json", JSON.parse(await readFile(originalInventory, "utf8"))) };
await requalifyNative(entry, "candidate", "7".repeat(64), "7".repeat(40));
const result = path.join(directory, "changed-proof-result.json");
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
`--original-h4e-proof=${entry.candidatePlaywrightFile}`, `--original-h4e-inventory=${entry.candidateInventoryFile}`, `--output=${result}`, "--require-acceptance"]);
assert.notEqual(attempt.status, 0);
const rejected = JSON.parse(await readFile(result, "utf8"));
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
assert.match(rejected.originalH4eConfirmation.reason, /Original confirmation used a different full qualified artifact inventory/);
});
const completionRegressionPairs = await manifest("verified-drain-regression", 1000, {
mutateArm: (report, { arm }) => {
if (arm !== "candidate") return;
@@ -424,7 +480,6 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
assert.equal(regressionGate.improvementRequired, true); assert.equal(regressionGate.passed, false);
assert(completionRegressionReport.gates.filter(gate => gate.name === "burst512" &&
["emissionToVisible.p95Ms", "intendedToVisible.p95Ms", "drainThroughTerminalMs"].includes(gate.metric)).every(gate => gate.passed));
const goodReport = JSON.parse(good.stdout);
const rate = goodReport.verifiedCompletionRates.find(item => item.name === "burst512");
assert.equal(rate.samples[0].measuredDeltas, 51200);
assert.equal(rate.samples[0].baselineDeltasPerSecond, 51200000 / 180010);
+20 -4
View File
@@ -1,4 +1,4 @@
# H5c canonical-settlement observation registration (prospective; no execution release)
# H5e matched-CF observation registration (prospective; no execution release)
H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5`
(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`).
@@ -8,14 +8,30 @@ concurrent remote participant and waits for the product's canonical settlement,
including supported same-run recovery, within the same terminal deadline.
It does not optimize the product or turn earlier failures into passes.
The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654`
and B1 `07061a601cdc20603c8c7a46f9ddbad7ac5f5939`. Raw B2
`ca7cba171cd5198ed569d75012edc858041cbc44` is the unoptimized batching base,
The aligned fresh application baselines are B0 `7550ffa5d6447d2615bc89fc5313f4cbb958b615`
and B1 `2cf1454406cbd582a11f04476713eceb746eada5`. Both normally include common
upstream `cf51936587963c26e3ddc7eb1365a07b2f8df5a6`. Raw B2
`0b616ccc90bf43cb26adac056a22ff01a84f982b` is the unoptimized batching base,
not a qualified future performance candidate. A later candidate needs its actual
new immutable revision, normal build receipt and inventory; no candidate reference
or matching binary is presumed before those artifacts exist. The historical
a365/d8cd/914 application failures remain evidence of their recorded revisions.
This prospective registration inherits H5d `2df9897e1ce2ca90be29fa097d776c3766e217ad`.
Only baseline input pins and the reviewed comparator binary-provenance correction
change: each arm must use its qualified Runner binary consistently, while A/A uses
one shared qualified build. Provider fixture hashes remain matched across arms.
The workload, oracle, deadlines, precision and suitability thresholds stay fixed.
The H5 registration digest changes because it includes the baseline references.
The old 4a83 campaign was operationally aborted after upstream changed exercised
remote checkpoint and ordinary UI Stop paths; preserve its completed/partial raw
attempts, cleanup failures and later owned cleanup separately. None supplies H5e
noise or a timing arm. Qualify the new actual normal builds, then collect fresh
main and companion B0/B1 A/A before further product optimization. No new build
receipt, native binary hash, inventory or execution release exists in this source
preparation. Historical H5b/c/d helpers remain immutable; their original comparator
cannot compare a distinct Rust B2 binary and must not execute this new campaign.
Select H5 explicitly with
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5c-observe-original-sla-independent-agents-settlement`.
The new default H4e mode uses the same independent-agent geometry with the original
+2 -2
View File
@@ -7,8 +7,8 @@ export const H4E_PROTOCOL = 'v4e-independent-agents-settlement-original-deadline
export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant',
fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50,
poolReuse: 'only-across-sequential-cases' });
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654',
B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' });
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: '7550ffa5d6447d2615bc89fc5313f4cbb958b615',
B1: '2cf1454406cbd582a11f04476713eceb746eada5' });
export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000,
streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 });
export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });