mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: freeze matched CF native ingestion inputs
This commit is contained in:
1 parent
2df9897e1c
commit
73ea851673
4 files changed
+91
-14
No files matched your search
@@ -281,9 +281,15 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
}
|
||||
assert.equal(remote.provider.fixtureSha256, report.fixtureSha256, "Staged provider fixture changed");
|
||||
assert.ok(remote.runner.metadata?.prpTransportModes?.includes("dial_wss"), "Remote daemon must support verified WSS");
|
||||
for (const [group, field] of [["runner", "sha256"], ["provider", "nodeSha256"], ["provider", "wrapperSha256"], ["provider", "fixtureSha256"]]) {
|
||||
assert.equal(remote[group][field], first.baseline.remote[group][field], "Remote provider/daemon artifact changed across matched samples");
|
||||
for (const field of ["nodeSha256", "wrapperSha256", "fixtureSha256"]) {
|
||||
assert.equal(remote.provider[field], first.baseline.remote.provider[field], "Remote provider artifact changed across matched samples");
|
||||
}
|
||||
// The candidate may change Runner code. Its serving hash is checked against
|
||||
// its own sealed qualified receipt by validateArmInventory before this loop.
|
||||
// Legacy comparisons without that receipt, and A/A, require one shared binary.
|
||||
const runnerReference = h5 && phase !== "aa" ? first[arm] : first.baseline;
|
||||
assert.equal(remote.runner.sha256, runnerReference.remote.runner.sha256, "Qualified Runner binary changed between samples of an arm");
|
||||
if (h5) assert.equal(remote.runner.sha256, report.armInventory.binarySha256, "Serving Runner differs from its arm's qualified build");
|
||||
assert.equal(report.admissionProtocol, "two-at-a-time-provider-and-clock-readiness-before-common-epoch", "Use the qualified pre-measurement admission protocol");
|
||||
assert.equal(report.setupActivationConcurrency, 2, "Keep the qualified setup activation concurrency of two");
|
||||
assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval");
|
||||
@@ -303,8 +309,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm");
|
||||
assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples");
|
||||
assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples");
|
||||
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples");
|
||||
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"])
|
||||
assert.equal(report.armInventory[field], first[phase === "aa" ? "baseline" : arm].armInventory[field], "Full qualified artifact inventory changed between samples");
|
||||
assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs");
|
||||
assert.match(report.harnessProvenance?.sourceRevision ?? "", /^[a-f0-9]{40}$/, "Missing frozen harness source revision");
|
||||
assert.equal(report.harnessProvenance.sourceStatus, "");
|
||||
@@ -455,7 +461,7 @@ if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
|
||||
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4e-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"] as const)
|
||||
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
|
||||
originalH4eConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision,
|
||||
retryObservations: proof.cases.map((item: any) => ({ name: item.name,
|
||||
@@ -629,7 +635,7 @@ if (phase === "compare") {
|
||||
if (h5) {
|
||||
assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256);
|
||||
assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required");
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision", "binarySha256", "rustSourceTree"])
|
||||
assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A");
|
||||
}
|
||||
assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol");
|
||||
|
||||
@@ -189,6 +189,7 @@ function browserReport(revision, latency) {
|
||||
|
||||
function h5Report(revision, loadedLatency) {
|
||||
const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000);
|
||||
if (revision === "B2") report.remote.runner.sha256 = "8".repeat(64);
|
||||
Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE,
|
||||
workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true,
|
||||
actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture",
|
||||
@@ -255,7 +256,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: {
|
||||
manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } });
|
||||
const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40),
|
||||
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: (variant === "B2" ? "8" : "9").repeat(40),
|
||||
binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }],
|
||||
artifactInventorySummary: inventorySummaryFile });
|
||||
inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile,
|
||||
@@ -292,6 +293,22 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
}
|
||||
return save(`${prefix}-pairs.json`, entries);
|
||||
}
|
||||
async function requalifyNative(entry, arm, binarySha256, rustSourceTree) {
|
||||
const report = JSON.parse(await readFile(entry[`${arm}File`], "utf8"));
|
||||
report.remote.runner.sha256 = binarySha256;
|
||||
await writeFile(entry[`${arm}File`], JSON.stringify(report));
|
||||
const owner = JSON.parse(await readFile(entry[`${arm}PlaywrightFile`], "utf8"));
|
||||
owner.suites[0].tests[0].results[0].attachments[0].body = Buffer.from(JSON.stringify(report)).toString("base64");
|
||||
await writeFile(entry[`${arm}PlaywrightFile`], JSON.stringify(owner));
|
||||
const sidecar = JSON.parse(await readFile(entry[`${arm}InventoryFile`], "utf8"));
|
||||
const receipt = JSON.parse(await readFile(sidecar.qualifiedReceiptFile, "utf8"));
|
||||
receipt.binarySha256 = binarySha256; receipt.rustSourceTree = rustSourceTree;
|
||||
receipt.nativeStageHashes[0].binarySha256 = binarySha256;
|
||||
sidecar.qualifiedReceiptFile = await save(`${path.basename(entry[`${arm}File`])}-qualified-native.json`, receipt);
|
||||
sidecar.qualifiedReceiptSha256 = hash(await readFile(sidecar.qualifiedReceiptFile));
|
||||
sidecar.playwrightReportSha256 = hash(await readFile(entry[`${arm}PlaywrightFile`]));
|
||||
await writeFile(entry[`${arm}InventoryFile`], JSON.stringify(sidecar));
|
||||
}
|
||||
try {
|
||||
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
|
||||
proof.protocolVersion = H4E_PROTOCOL;
|
||||
@@ -403,6 +420,45 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
const goodPairs = await manifest("good", 1000);
|
||||
const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(good.status, 0, good.stderr);
|
||||
const goodReport = JSON.parse(good.stdout);
|
||||
assert.equal(goodReport.artifactInventories.baseline.binarySha256, "d".repeat(64));
|
||||
assert.equal(goodReport.artifactInventories.candidate.binarySha256, "8".repeat(64));
|
||||
assert.notEqual(goodReport.artifactInventories.baseline.rustSourceTree, goodReport.artifactInventories.candidate.rustSourceTree);
|
||||
await context.test("each arm accepts its distinct qualified Runner binary", () => {
|
||||
assert.equal(goodReport.performancePassed, true); assert.equal(goodReport.acceptancePassed, true);
|
||||
assert.equal(goodReport.originalH4eConfirmation.inventory.binarySha256, "8".repeat(64));
|
||||
});
|
||||
await context.test("a serving binary outside its qualified receipt is rejected", async () => {
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${await manifest("unqualified-native", 1000, {
|
||||
mutateArm: (report, { arm, index }) => { if (arm === "candidate" && index === 2) report.remote.runner.sha256 = "7".repeat(64); },
|
||||
})}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Serving native binary differs from qualified build/);
|
||||
});
|
||||
await context.test("a second qualified binary within one arm is rejected", async () => {
|
||||
const file = await manifest("changed-qualified-native", 1000), entries = JSON.parse(await readFile(file, "utf8"));
|
||||
await requalifyNative(entries[2], "candidate", "7".repeat(64), "7".repeat(40));
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${file}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/);
|
||||
});
|
||||
await context.test("A/A cannot compare two qualified Runner binaries", async () => {
|
||||
const file = await manifest("aa-changed-qualified-native", 181000, { sameRevision: true }), entries = JSON.parse(await readFile(file, "utf8"));
|
||||
await requalifyNative(entries[0], "candidate", "7".repeat(64), "7".repeat(40));
|
||||
const attempt = run(["--phase=aa", `--pairs=${file}`]);
|
||||
assert.notEqual(attempt.status, 0); assert.match(attempt.stderr, /Qualified Runner binary changed between samples of an arm/);
|
||||
});
|
||||
await context.test("later original-deadline proof must use the candidate's qualified binary", async () => {
|
||||
const entry = { candidateFile: await save("changed-proof-evidence.json", proof),
|
||||
candidatePlaywrightFile: await save("changed-proof-playwright.json", JSON.parse(await readFile(originalProof, "utf8"))),
|
||||
candidateInventoryFile: await save("changed-proof-inventory.json", JSON.parse(await readFile(originalInventory, "utf8"))) };
|
||||
await requalifyNative(entry, "candidate", "7".repeat(64), "7".repeat(40));
|
||||
const result = path.join(directory, "changed-proof-result.json");
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
|
||||
`--original-h4e-proof=${entry.candidatePlaywrightFile}`, `--original-h4e-inventory=${entry.candidateInventoryFile}`, `--output=${result}`, "--require-acceptance"]);
|
||||
assert.notEqual(attempt.status, 0);
|
||||
const rejected = JSON.parse(await readFile(result, "utf8"));
|
||||
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
|
||||
assert.match(rejected.originalH4eConfirmation.reason, /Original confirmation used a different full qualified artifact inventory/);
|
||||
});
|
||||
const completionRegressionPairs = await manifest("verified-drain-regression", 1000, {
|
||||
mutateArm: (report, { arm }) => {
|
||||
if (arm !== "candidate") return;
|
||||
@@ -424,7 +480,6 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
assert.equal(regressionGate.improvementRequired, true); assert.equal(regressionGate.passed, false);
|
||||
assert(completionRegressionReport.gates.filter(gate => gate.name === "burst512" &&
|
||||
["emissionToVisible.p95Ms", "intendedToVisible.p95Ms", "drainThroughTerminalMs"].includes(gate.metric)).every(gate => gate.passed));
|
||||
const goodReport = JSON.parse(good.stdout);
|
||||
const rate = goodReport.verifiedCompletionRates.find(item => item.name === "burst512");
|
||||
assert.equal(rate.samples[0].measuredDeltas, 51200);
|
||||
assert.equal(rate.samples[0].baselineDeltasPerSecond, 51200000 / 180010);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# H5c canonical-settlement observation registration (prospective; no execution release)
|
||||
# H5e matched-CF observation registration (prospective; no execution release)
|
||||
|
||||
H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5`
|
||||
(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`).
|
||||
@@ -8,14 +8,30 @@ concurrent remote participant and waits for the product's canonical settlement,
|
||||
including supported same-run recovery, within the same terminal deadline.
|
||||
It does not optimize the product or turn earlier failures into passes.
|
||||
|
||||
The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654`
|
||||
and B1 `07061a601cdc20603c8c7a46f9ddbad7ac5f5939`. Raw B2
|
||||
`ca7cba171cd5198ed569d75012edc858041cbc44` is the unoptimized batching base,
|
||||
The aligned fresh application baselines are B0 `7550ffa5d6447d2615bc89fc5313f4cbb958b615`
|
||||
and B1 `2cf1454406cbd582a11f04476713eceb746eada5`. Both normally include common
|
||||
upstream `cf51936587963c26e3ddc7eb1365a07b2f8df5a6`. Raw B2
|
||||
`0b616ccc90bf43cb26adac056a22ff01a84f982b` is the unoptimized batching base,
|
||||
not a qualified future performance candidate. A later candidate needs its actual
|
||||
new immutable revision, normal build receipt and inventory; no candidate reference
|
||||
or matching binary is presumed before those artifacts exist. The historical
|
||||
a365/d8cd/914 application failures remain evidence of their recorded revisions.
|
||||
|
||||
This prospective registration inherits H5d `2df9897e1ce2ca90be29fa097d776c3766e217ad`.
|
||||
Only baseline input pins and the reviewed comparator binary-provenance correction
|
||||
change: each arm must use its qualified Runner binary consistently, while A/A uses
|
||||
one shared qualified build. Provider fixture hashes remain matched across arms.
|
||||
The workload, oracle, deadlines, precision and suitability thresholds stay fixed.
|
||||
The H5 registration digest changes because it includes the baseline references.
|
||||
The old 4a83 campaign was operationally aborted after upstream changed exercised
|
||||
remote checkpoint and ordinary UI Stop paths; preserve its completed/partial raw
|
||||
attempts, cleanup failures and later owned cleanup separately. None supplies H5e
|
||||
noise or a timing arm. Qualify the new actual normal builds, then collect fresh
|
||||
main and companion B0/B1 A/A before further product optimization. No new build
|
||||
receipt, native binary hash, inventory or execution release exists in this source
|
||||
preparation. Historical H5b/c/d helpers remain immutable; their original comparator
|
||||
cannot compare a distinct Rust B2 binary and must not execute this new campaign.
|
||||
|
||||
Select H5 explicitly with
|
||||
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5c-observe-original-sla-independent-agents-settlement`.
|
||||
The new default H4e mode uses the same independent-agent geometry with the original
|
||||
|
||||
@@ -7,8 +7,8 @@ export const H4E_PROTOCOL = 'v4e-independent-agents-settlement-original-deadline
|
||||
export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant',
|
||||
fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50,
|
||||
poolReuse: 'only-across-sequential-cases' });
|
||||
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654',
|
||||
B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' });
|
||||
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: '7550ffa5d6447d2615bc89fc5313f4cbb958b615',
|
||||
B1: '2cf1454406cbd582a11f04476713eceb746eada5' });
|
||||
export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000,
|
||||
streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 });
|
||||
export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });
|
||||
|
||||
Reference in new issue
Block a user