mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: observe full ingestion drainage with original deadline outcomes
This commit is contained in:
1 parent
7d7e7a88d5
commit
81c9ffdf03
10 files changed
+1120
-14
No files matched your search
@@ -3,6 +3,8 @@ import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
import { H5_MODE, H5_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs";
|
||||
|
||||
// Each manifest entry points at native-ingestion-evidence JSON attachments
|
||||
// from two isolated compiled-app runs. Run the variants in alternating order;
|
||||
@@ -11,12 +13,87 @@ const arg = (name: string) => process.argv.find(value => value.startsWith(`--${n
|
||||
const manifestPath = arg("pairs");
|
||||
assert.ok(manifestPath, "Pass --pairs=<JSON manifest of baselineFile/candidateFile pairs>");
|
||||
const phase = arg("phase") ?? "compare", expectation = arg("expect") ?? "improvement";
|
||||
const h5 = arg("observation-mode") === H5_MODE;
|
||||
assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode");
|
||||
const candidatePolicy = arg("candidate-policy");
|
||||
assert.ok(["aa", "compare"].includes(phase));
|
||||
assert.ok(["improvement", "nonregression"].includes(expectation));
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string }>;
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string;
|
||||
baselinePlaywrightFile?: string; candidatePlaywrightFile?: string; baselineInventoryFile?: string; candidateInventoryFile?: string }>;
|
||||
assert.ok(manifest.length >= 5, "At least five actual paired application runs are required");
|
||||
if (h5) assert.equal(manifest.length, 5, "H5 requires exactly five prospectively registered pairs");
|
||||
const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8"));
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) })));
|
||||
async function owningResult(reportPath: string, evidence?: any) {
|
||||
const bytes = await readFile(reportPath);
|
||||
const report = JSON.parse(bytes.toString("utf8")), tests: any[] = [];
|
||||
assert.deepEqual(report.errors ?? [], [], "Owning Playwright report has a global error");
|
||||
const visit = (value: any) => {
|
||||
if (Array.isArray(value)) for (const child of value) visit(child);
|
||||
else if (value && typeof value === "object") {
|
||||
if (Array.isArray(value.results) && Object.hasOwn(value, "expectedStatus")) tests.push(value);
|
||||
for (const [key, child] of Object.entries(value)) if (key !== "results") visit(child);
|
||||
}
|
||||
};
|
||||
visit(report); assert.equal(tests.length, 1, "One complete owning test is required; no partial selection");
|
||||
const test = tests[0]; assert.equal(test.results.length, 1, "No retry or omitted owning attempt");
|
||||
assert.equal(test.expectedStatus, "passed");
|
||||
const result = test.results[0], attachments = result.attachments.filter((item: any) => item.name === "native-ingestion-evidence");
|
||||
assert.ok(result.error === undefined || result.error === null, "Owning Playwright attempt has an error");
|
||||
assert.equal(attachments.length, 1, "Owning result must carry this exact evidence attachment");
|
||||
const attachment = attachments[0];
|
||||
const attached = attachment.body ? JSON.parse(Buffer.from(attachment.body, "base64").toString("utf8"))
|
||||
: JSON.parse(await readFile(path.resolve(path.dirname(reportPath), attachment.path), "utf8"));
|
||||
if (evidence) assert.deepEqual(attached, evidence, "Playwright result belongs to a different evidence arm");
|
||||
return { evidence: attached, owner: { reportFile: reportPath, reportSha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
status: result.status, retry: result.retry, errors: result.errors, startedAt: result.startTime,
|
||||
durationMs: result.duration, timeoutMs: test.timeout } };
|
||||
}
|
||||
async function loadArm(file: string, playwrightFile?: string, inventoryFile?: string) {
|
||||
const evidence = await load(file);
|
||||
if (!h5) return evidence;
|
||||
assert.ok(playwrightFile, "H5 requires each arm's independent owning Playwright JSON report");
|
||||
evidence.owningPlaywright = (await owningResult(path.resolve(path.dirname(manifestPath!), playwrightFile), evidence)).owner;
|
||||
assert.ok(inventoryFile, "H5 requires each arm's exact full artifact inventory sidecar");
|
||||
evidence.armInventory = await validateArmInventory(path.resolve(path.dirname(manifestPath!), inventoryFile),
|
||||
{ evidence, owningPlaywright: evidence.owningPlaywright });
|
||||
return evidence;
|
||||
}
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await loadArm(entry.baselineFile, entry.baselinePlaywrightFile, entry.baselineInventoryFile),
|
||||
candidate: await loadArm(entry.candidateFile, entry.candidatePlaywrightFile, entry.candidateInventoryFile) })));
|
||||
const h5Assessments: any[] = [];
|
||||
if (h5) {
|
||||
if (phase === "compare") {
|
||||
assert.ok(["final-b2", "cursor-nonregression"].includes(candidatePolicy ?? ""), "H5 requires an explicit candidate SLA policy");
|
||||
assert.equal(expectation, candidatePolicy === "cursor-nonregression" ? "nonregression" : "improvement");
|
||||
}
|
||||
for (const pair of pairs) {
|
||||
assert.ok(Object.values(H5_BASELINE_REVISIONS).includes(pair.baseline.sourceRevision), "H5 baseline must use the prospectively frozen B0 or B1 revision");
|
||||
const baselineVariant = pair.baseline.sourceRevision === H5_BASELINE_REVISIONS.B0 ? "B0" : "B1";
|
||||
assert.equal(pair.baseline.armInventory.variant, baselineVariant, "Baseline inventory role differs from its frozen revision");
|
||||
if (phase === "aa") assert.equal(pair.candidate.armInventory.variant, baselineVariant, "A/A must use the same baseline inventory role");
|
||||
if (phase === "compare" && candidatePolicy === "cursor-nonregression") {
|
||||
assert.equal(pair.baseline.sourceRevision, H5_BASELINE_REVISIONS.B0, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.sourceRevision, H5_BASELINE_REVISIONS.B1, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.armInventory.variant, "B1", "Cursor policy cannot stand in for final B2 acceptance");
|
||||
} else if (phase === "compare") {
|
||||
assert.equal(pair.candidate.armInventory.variant, "B2", "Final policy requires the qualified B2 inventory role");
|
||||
assert.ok(!Object.values(H5_BASELINE_REVISIONS).includes(pair.candidate.sourceRevision), "Final B2 must differ from both frozen baselines");
|
||||
}
|
||||
const baseline = assessH5Arm(pair.baseline, pair.baseline.owningPlaywright);
|
||||
const candidate = assessH5Arm(pair.candidate, pair.candidate.owningPlaywright);
|
||||
// The only prospectively known baseline workload SLA miss is loaded streaming.
|
||||
const allowed = new Set(["burst512:streaming120", "invocation:observed-total-duration900"]);
|
||||
for (const miss of baseline.misses) assert.ok(allowed.has(miss), "Unknown baseline deadline failure needs diagnosis");
|
||||
if (phase === "aa") for (const miss of candidate.misses) assert.ok(allowed.has(miss), "Unknown A/A deadline failure needs diagnosis");
|
||||
let candidateSlaPolicyError;
|
||||
if (phase === "compare") {
|
||||
try { requireH5Candidate(candidatePolicy, baseline, candidate); }
|
||||
catch (error) { candidateSlaPolicyError = error instanceof Error ? error.message : String(error); }
|
||||
}
|
||||
h5Assessments.push({ baseline, candidate, candidateSlaPolicyError, candidateSlaPolicyPassed: !candidateSlaPolicyError,
|
||||
owningPlaywright: { baseline: pair.baseline.owningPlaywright, candidate: pair.candidate.owningPlaywright } });
|
||||
}
|
||||
}
|
||||
const workloads = [
|
||||
{ name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
@@ -32,7 +109,8 @@ const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/
|
||||
const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
|
||||
for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
|
||||
const report = pair[arm];
|
||||
assert.equal(report.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream", "Fresh browser acceptance requires the qualified v4 mixed clock protocol");
|
||||
assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : "v4-remote-streaming-original-local-stop-clock-stream",
|
||||
h5 ? "Fresh browser acceptance requires its qualified H5 observation protocol" : "Fresh browser acceptance requires the qualified v4 mixed clock protocol");
|
||||
assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport");
|
||||
assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, "Missing explicit transport and real cloud Stop qualification policy");
|
||||
@@ -97,6 +175,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm");
|
||||
assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples");
|
||||
assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples");
|
||||
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples");
|
||||
assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs");
|
||||
if (phase === "aa") assert.equal(report.sourceRevision, first.baseline.sourceRevision, "A/A must use one source revision");
|
||||
for (const name of names) {
|
||||
@@ -300,6 +380,56 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`);
|
||||
}
|
||||
}
|
||||
let originalH4cConfirmation: Record<string, any> | undefined;
|
||||
if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
originalH4cConfirmation = { required: true, passed: false, reason: "Later unchanged original H4c full functional proof is pending" };
|
||||
if (arg("original-h4c-proof")) {
|
||||
try {
|
||||
assert.ok(arg("original-h4c-inventory"), "Original H4c proof requires its full artifact inventory sidecar");
|
||||
const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4c-proof")!));
|
||||
assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.equal(proof.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream");
|
||||
assert.equal(proof.observationMode, undefined, "Confirmation must use unchanged original H4c defaults");
|
||||
assert.equal(proof.harnessProvenance?.sourceRevision, "7d7e7a88d562a95a1d407b76474763b666c181f5");
|
||||
assert.equal(proof.harnessProvenance.sourceStatus, "");
|
||||
assert.equal(proof.measurementSha256, "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b");
|
||||
assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false);
|
||||
assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled");
|
||||
assert.equal(proof.cleanup?.complete, true); assert.deepEqual(proof.cleanup.errors, []);
|
||||
assert.deepEqual(proof.browserRecording, first.candidate.browserRecording);
|
||||
assert.equal(proof.sourceRevision, first.candidate.sourceRevision); assert.equal(proof.compiledBuildRevision, first.candidate.compiledBuildRevision);
|
||||
assert.equal(proof.serverEntrySha256, first.candidate.serverEntrySha256);
|
||||
assert.deepEqual(proof.compiledHotPathHashes, first.candidate.compiledHotPathHashes);
|
||||
assert.equal(proof.transport, first.candidate.transport); assert.deepEqual(proof.transportPolicy, first.candidate.transportPolicy);
|
||||
assert.deepEqual(proof.cases.map((item: any) => item.name), names, "Original confirmation must include all six workloads");
|
||||
for (const fixed of workloads) {
|
||||
const item = proof.cases.find((item: any) => item.name === fixed.name);
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(item[field], fixed[field]);
|
||||
assert.equal(item.persistedAfterReload, true); assert.equal(item.durable.length, fixed.runs);
|
||||
assert.equal(item.matchedVisibleClocks, item.visible.length);
|
||||
assert.ok(item.visible.length >= (fixed.stop ? 3 : fixed.events));
|
||||
assert.equal(item.runIds.length, fixed.runs); assert.equal(new Set(item.runIds).size, fixed.runs);
|
||||
for (const row of item.durable) {
|
||||
assert.equal(row.ordered, true); assert.equal(row.uniqueSourceIdentities, row.individualDeltas);
|
||||
assert.equal(row.matchedSourceClocks, row.individualDeltas);
|
||||
if (!fixed.stop) assert.equal(row.individualDeltas, fixed.events);
|
||||
}
|
||||
}
|
||||
const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs));
|
||||
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
|
||||
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4c-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
|
||||
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
|
||||
originalH4cConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision };
|
||||
} catch (error) {
|
||||
originalH4cConfirmation.reason = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
const metric = (item: any, key: string): number => key.split(".").reduce((value, part) => value[part], item);
|
||||
const measurements = names.flatMap(name => {
|
||||
const observed = first.baseline.cases.find((item: any) => item.name === name);
|
||||
@@ -334,11 +464,21 @@ const result: Record<string, any> = {
|
||||
comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
|
||||
manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements,
|
||||
primaryLoadedWorkload: "burst512", complete: true,
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments,
|
||||
artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4cConfirmation,
|
||||
originalProtocolAcceptance: "Baseline SLA misses are retained; H5 complete observation is not H4c PASS",
|
||||
finalUnchangedH4cFunctionalPassRequired: candidatePolicy === "final-b2" } : {}),
|
||||
};
|
||||
if (phase === "compare") {
|
||||
assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report");
|
||||
const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8"));
|
||||
assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true);
|
||||
if (h5) {
|
||||
assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256);
|
||||
assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required");
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A");
|
||||
}
|
||||
assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol");
|
||||
assert.equal(noise.transport, result.transport, "Noise report uses a different transport");
|
||||
assert.deepEqual(noise.transportPolicy, result.transportPolicy, "Transport policy changed after A/A calibration");
|
||||
@@ -362,10 +502,17 @@ if (phase === "compare") {
|
||||
noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
|
||||
passed: improvementRequired ? improvement : noRegression };
|
||||
});
|
||||
result.acceptancePassed = result.gates.every((entry: any) => entry.passed);
|
||||
result.performancePassed = result.gates.every((entry: any) => entry.passed)
|
||||
&& (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed));
|
||||
result.acceptancePassed = result.performancePassed && (!originalH4cConfirmation || originalH4cConfirmation.passed);
|
||||
}
|
||||
if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (process.argv.includes("--require-performance")) {
|
||||
assert.equal(h5, true, "H5 performance-only gating is explicit; original H4c remains unchanged");
|
||||
assert.equal(phase, "compare");
|
||||
assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose");
|
||||
}
|
||||
if (process.argv.includes("--require-acceptance")) {
|
||||
assert.equal(phase, "compare", "A/A calibrates noise; it is not performance acceptance");
|
||||
assert.equal(result.acceptancePassed, true, "Browser performance acceptance failed; retain results and diagnose");
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { H5_REGISTRATION, H5_WORKLOAD_SHA256 } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
if (process.argv.length > 2) throw new Error('Dry run only: this command accepts no execution arguments');
|
||||
console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadSha256: H5_WORKLOAD_SHA256,
|
||||
registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes',
|
||||
'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation',
|
||||
'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass',
|
||||
'Later unchanged original H4c full functional pass on the final SUT'] }, null, 2));
|
||||
@@ -0,0 +1,137 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
const SCHEMA = 'paperclip.native-ingestion.arm-inventory.v1';
|
||||
const ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const sha256 = value => createHash('sha256').update(value).digest('hex');
|
||||
const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
||||
const revision = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
|
||||
const object = value => value && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
async function bounded(file, maximumBytes) {
|
||||
const metadata = await stat(file);
|
||||
assert.ok(metadata.isFile() && metadata.size <= maximumBytes, 'Inventory evidence file is absent, nonregular or oversized');
|
||||
const bytes = await readFile(file);
|
||||
assert.ok(bytes.length <= maximumBytes, 'Inventory evidence file grew beyond its bound');
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function relativeArtifact(value) {
|
||||
return typeof value === 'string' && value.length > 0 && !value.includes('\\')
|
||||
&& !value.includes('\0') && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value
|
||||
&& !value.split('/').includes('..') && ROOTS.some(root => value.startsWith(`${root}/`));
|
||||
}
|
||||
|
||||
// The existing Python generator sorts Unicode code points and uses json.dumps
|
||||
// with sort_keys=True, separators=(',', ':') and its default ensure_ascii=True.
|
||||
function pythonOrder(left, right) {
|
||||
const a = [...left], b = [...right];
|
||||
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
||||
const difference = a[index].codePointAt(0) - b[index].codePointAt(0);
|
||||
if (difference) return difference;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })
|
||||
.replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
||||
|
||||
/**
|
||||
* Validates frozen inventory evidence and reported serving fingerprints only.
|
||||
* The owner supplies this sidecar per arm; this helper never reads the SUT tree
|
||||
* and does not attest that a live filesystem remained unchanged during a run.
|
||||
*/
|
||||
export async function validateArmInventory(sidecarFile, { evidence, owningPlaywright }) {
|
||||
assert.equal(typeof sidecarFile, 'string', 'Missing arm inventory sidecar');
|
||||
assert.ok(object(evidence) && object(owningPlaywright), 'Missing serving evidence or owning Playwright report');
|
||||
const absoluteSidecar = path.resolve(sidecarFile), base = path.dirname(absoluteSidecar);
|
||||
const sidecarBytes = await bounded(absoluteSidecar, 256 * 1024);
|
||||
const sidecar = JSON.parse(sidecarBytes.toString('utf8'));
|
||||
assert.equal(sidecar.schema, SCHEMA, 'Unsupported arm inventory sidecar schema');
|
||||
assert.ok(['B0', 'B1', 'B2'].includes(sidecar.variant), 'Unknown inventory variant');
|
||||
assert.ok(revision(sidecar.sourceRevision) && revision(sidecar.compiledBuildRevision), 'Invalid inventory source revision');
|
||||
assert.equal(sidecar.sourceRevision, evidence.sourceRevision, 'Inventory source revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, evidence.compiledBuildRevision, 'Inventory build revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, sidecar.sourceRevision, 'Inventory was built from a different source revision');
|
||||
assert.ok(digest(sidecar.playwrightReportSha256) && digest(owningPlaywright.reportSha256), 'Missing owning Playwright report hash');
|
||||
assert.equal(sidecar.playwrightReportSha256, owningPlaywright.reportSha256, 'Inventory belongs to another owning Playwright report');
|
||||
const resolve = name => {
|
||||
assert.ok(typeof sidecar[name] === 'string' && sidecar[name].length > 0, `Missing inventory ${name}`);
|
||||
return path.resolve(base, sidecar[name]);
|
||||
};
|
||||
const readHashed = async (name, hashName, maximumBytes) => {
|
||||
assert.ok(digest(sidecar[hashName]), `Invalid inventory ${hashName}`);
|
||||
const file = resolve(name), bytes = await bounded(file, maximumBytes);
|
||||
assert.equal(sha256(bytes), sidecar[hashName], `Inventory ${name} bytes changed`);
|
||||
return { file, bytes };
|
||||
};
|
||||
const manifestSource = await readHashed('manifestFile', 'manifestFileSha256', 64 * 1024 * 1024);
|
||||
const manifest = JSON.parse(gunzipSync(manifestSource.bytes, { maxOutputLength: 128 * 1024 * 1024 }).toString('utf8'));
|
||||
assert.deepEqual(Object.keys(manifest).sort(), ['aggregateSha256', 'compiledBuildRevision', 'fileCount', 'files', 'revision', 'roots', 'variant']);
|
||||
assert.equal(manifest.variant, sidecar.variant, 'Inventory manifest variant differs from sidecar');
|
||||
assert.equal(manifest.revision, sidecar.sourceRevision, 'Inventory manifest source revision differs from arm');
|
||||
assert.equal(manifest.compiledBuildRevision, sidecar.compiledBuildRevision, 'Inventory manifest build revision differs from arm');
|
||||
assert.deepEqual(manifest.roots, ROOTS, 'Full registered inventory roots are required');
|
||||
assert.ok(Number.isSafeInteger(manifest.fileCount) && manifest.fileCount > 0 && manifest.fileCount <= 500000, 'Invalid inventory file count');
|
||||
assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount, 'Inventory file count is incomplete');
|
||||
assert.ok(digest(manifest.aggregateSha256) && digest(sidecar.aggregateSha256), 'Invalid inventory aggregate hash');
|
||||
const files = new Map();
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(object(row), 'Invalid inventory file row');
|
||||
assert.deepEqual(Object.keys(row).sort(), ['bytes', 'path', 'sha256']);
|
||||
assert.ok(relativeArtifact(row.path), 'Inventory file escapes registered roots');
|
||||
assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0 && digest(row.sha256), 'Invalid inventory file descriptor');
|
||||
assert.equal(files.has(row.path), false, 'Duplicate inventory file path');
|
||||
files.set(row.path, row);
|
||||
}
|
||||
const sorted = [...manifest.files].sort((left, right) => pythonOrder(left.path, right.path));
|
||||
assert.deepEqual(manifest.files.map(row => row.path), sorted.map(row => row.path), 'Inventory file order differs from generator');
|
||||
for (const root of ROOTS) assert.ok(manifest.files.some(row => row.path.startsWith(`${root}/`)), 'Inventory omitted a registered root');
|
||||
const aggregate = sha256(Buffer.from(`${sorted.map(pythonRow).join('\n')}\n`, 'utf8'));
|
||||
assert.equal(aggregate, manifest.aggregateSha256, 'Inventory aggregate does not match exact file descriptors');
|
||||
assert.equal(aggregate, sidecar.aggregateSha256, 'Inventory aggregate differs from sidecar');
|
||||
assert.ok(digest(evidence.serverEntrySha256), 'Missing serving server entry hash');
|
||||
assert.equal(files.get('server/dist/index.js')?.sha256, evidence.serverEntrySha256, 'Serving server entry differs from full inventory');
|
||||
assert.ok(object(evidence.compiledHotPathHashes) && Object.keys(evidence.compiledHotPathHashes).length > 0, 'Missing serving hot-path fingerprints');
|
||||
for (const [file, hash] of Object.entries(evidence.compiledHotPathHashes)) {
|
||||
assert.ok(relativeArtifact(file) && digest(hash), 'Invalid serving hot-path fingerprint');
|
||||
assert.equal(files.get(file)?.sha256, hash, 'Serving hot path differs from full inventory');
|
||||
}
|
||||
const summarySource = await readHashed('inventorySummaryFile', 'inventorySummarySha256', 1024 * 1024);
|
||||
const summary = JSON.parse(summarySource.bytes.toString('utf8'));
|
||||
const summaryRow = summary.inventories?.[sidecar.variant];
|
||||
assert.ok(object(summaryRow), 'Inventory summary omitted this variant');
|
||||
assert.ok(typeof summaryRow.manifest === 'string' && path.basename(summaryRow.manifest) === summaryRow.manifest
|
||||
&& summaryRow.manifest.endsWith('.json.gz'), 'Inventory summary manifest reference is invalid');
|
||||
assert.equal(summaryRow.manifestSha256, sidecar.manifestFileSha256, 'Inventory summary points to different manifest bytes');
|
||||
assert.equal(summaryRow.aggregateSha256, aggregate, 'Inventory summary aggregate differs');
|
||||
assert.equal(summaryRow.fileCount, manifest.fileCount, 'Inventory summary file count differs');
|
||||
const receiptSource = await readHashed('qualifiedReceiptFile', 'qualifiedReceiptSha256', 4 * 1024 * 1024);
|
||||
const receipt = JSON.parse(receiptSource.bytes.toString('utf8'));
|
||||
assert.ok(Array.isArray(receipt.results), 'Qualified build receipt has no results');
|
||||
const qualified = receipt.results.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(qualified.length, 1, 'Qualified build receipt must identify this variant exactly once');
|
||||
assert.equal(qualified[0].revision, sidecar.sourceRevision, 'Qualified build receipt source revision differs');
|
||||
assert.equal(qualified[0].compiledBuildRevision, sidecar.compiledBuildRevision, 'Qualified build receipt build revision differs');
|
||||
assert.equal(qualified[0].sourceClean, true, 'Qualified build source was not clean');
|
||||
assert.ok(typeof receipt.artifactInventorySummary === 'string' && receipt.artifactInventorySummary.length > 0,
|
||||
'Qualified build receipt omitted its inventory summary reference');
|
||||
assert.ok(revision(receipt.rustSourceTree), 'Qualified build receipt omitted Rust input identity');
|
||||
assert.ok(Array.isArray(receipt.nativeStageHashes), 'Qualified build receipt omitted native stage hashes');
|
||||
const native = receipt.nativeStageHashes.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(native.length, 1, 'Qualified build receipt must identify this native stage exactly once');
|
||||
assert.ok(digest(native[0].binarySha256) && digest(receipt.binarySha256), 'Qualified native binary hash is invalid');
|
||||
assert.equal(native[0].binarySha256, receipt.binarySha256, 'Qualified native binary hashes disagree');
|
||||
assert.equal(native[0].binarySha256, evidence.remote?.runner?.sha256, 'Serving native binary differs from qualified build');
|
||||
return { schema: SCHEMA, sidecarFile: absoluteSidecar, sidecarFileSha256: sha256(sidecarBytes), variant: sidecar.variant,
|
||||
revision: manifest.revision, compiledBuildRevision: manifest.compiledBuildRevision,
|
||||
manifestFile: manifestSource.file, manifestFileSha256: sidecar.manifestFileSha256,
|
||||
aggregateSha256: aggregate, fileCount: manifest.fileCount, roots: manifest.roots,
|
||||
inventorySummaryFile: summarySource.file, inventorySummarySha256: sidecar.inventorySummarySha256,
|
||||
qualifiedReceiptFile: receiptSource.file, qualifiedReceiptSha256: sidecar.qualifiedReceiptSha256,
|
||||
binarySha256: native[0].binarySha256, rustSourceTree: receipt.rustSourceTree,
|
||||
playwrightReportSha256: owningPlaywright.reportSha256,
|
||||
scope: 'Exact frozen manifest/summary/build-receipt bytes for all registered roots, with reported server/hot-path/native fingerprints checked; no live SUT filesystem attestation' };
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import { validateArmInventory } from './native-ingestion-h5-inventory.mjs';
|
||||
|
||||
const hash = value => createHash('sha256').update(value).digest('hex');
|
||||
const sourceRevision = '1'.repeat(40), reportSha256 = hash('owning-playwright'), binarySha256 = hash('runner');
|
||||
const roots = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const descriptors = () => [
|
||||
'packages/db/src/client.ts', 'packages/shared/src/index.ts', 'server/dist/hot-a.js',
|
||||
'server/dist/hot-b.js', 'server/dist/hot-c.js', 'server/dist/index.js', 'ui/dist/index.html',
|
||||
].map(file => ({ bytes: 10, path: file, sha256: hash(file) }));
|
||||
const aggregate = files => hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join('\n') + '\n');
|
||||
|
||||
async function fixture(directory, options = {}) {
|
||||
const revision = options.revision ?? sourceRevision;
|
||||
const files = descriptors(), manifest = { variant: 'B2', revision, compiledBuildRevision: revision,
|
||||
roots, fileCount: files.length, aggregateSha256: aggregate(files), files };
|
||||
options.manifest?.(manifest);
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify(manifest)));
|
||||
const summary = { capturedAt: '2026-11-03T00:00:00.000Z', inventories: { B2: {
|
||||
manifest: 'B2-artifact-inventory.json.gz', manifestSha256: hash(manifestBytes), fileCount: manifest.fileCount,
|
||||
aggregateSha256: manifest.aggregateSha256 } } };
|
||||
options.summary?.(summary);
|
||||
const receipt = { completedAt: '2026-11-03T00:00:01.000Z',
|
||||
results: [{ variant: 'B2', revision, compiledBuildRevision: revision, sourceClean: true }],
|
||||
rustSourceTree: '2'.repeat(40), binarySha256, nativeStageHashes: [{ variant: 'B2', binarySha256 }],
|
||||
artifactInventorySummary: '/owned/original/artifact-inventory-summary.json' };
|
||||
options.receipt?.(receipt);
|
||||
const summaryBytes = Buffer.from(JSON.stringify(summary)), receiptBytes = Buffer.from(JSON.stringify(receipt));
|
||||
const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant: 'B2', sourceRevision: revision,
|
||||
compiledBuildRevision: revision, playwrightReportSha256: reportSha256,
|
||||
manifestFile: 'B2-artifact-inventory.json.gz', manifestFileSha256: hash(manifestBytes),
|
||||
aggregateSha256: manifest.aggregateSha256,
|
||||
qualifiedReceiptFile: 'qualified.json', qualifiedReceiptSha256: hash(receiptBytes),
|
||||
inventorySummaryFile: 'artifact-inventory-summary.json', inventorySummarySha256: hash(summaryBytes) };
|
||||
options.sidecar?.(sidecar);
|
||||
const evidence = { sourceRevision: revision, compiledBuildRevision: revision,
|
||||
serverEntrySha256: hash('server/dist/index.js'), remote: { runner: { sha256: binarySha256 } },
|
||||
compiledHotPathHashes: Object.fromEntries(['server/dist/hot-a.js', 'server/dist/hot-b.js', 'server/dist/hot-c.js'].map(file => [file, hash(file)])) };
|
||||
options.evidence?.(evidence);
|
||||
await Promise.all([
|
||||
writeFile(path.join(directory, sidecar.manifestFile), manifestBytes),
|
||||
writeFile(path.join(directory, sidecar.inventorySummaryFile), summaryBytes),
|
||||
writeFile(path.join(directory, sidecar.qualifiedReceiptFile), receiptBytes),
|
||||
writeFile(path.join(directory, 'arm-inventory.json'), JSON.stringify(sidecar)),
|
||||
]);
|
||||
return { sidecarFile: path.join(directory, 'arm-inventory.json'), sidecar, evidence,
|
||||
owningPlaywright: { reportSha256 }, manifest, summary, receipt };
|
||||
}
|
||||
|
||||
async function withFixture(options, work) {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'h5-inventory-'));
|
||||
try { await work(await fixture(directory, options), directory); }
|
||||
finally { await rm(directory, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
test('full frozen inventory, qualified native build and owning report bind one arm', async () => {
|
||||
await withFixture({}, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.manifestFileSha256, value.sidecar.manifestFileSha256);
|
||||
assert.equal(proof.aggregateSha256, aggregate(descriptors()));
|
||||
assert.equal(proof.fileCount, descriptors().length);
|
||||
assert.equal(proof.qualifiedReceiptSha256, value.sidecar.qualifiedReceiptSha256);
|
||||
assert.equal(proof.playwrightReportSha256, reportSha256);
|
||||
assert.equal(proof.binarySha256, binarySha256);
|
||||
assert.deepEqual(proof.roots, roots);
|
||||
});
|
||||
});
|
||||
|
||||
test('exact compressed manifest bytes and build-receipt bytes cannot be swapped', async () => {
|
||||
for (const field of ['manifestFile', 'qualifiedReceiptFile']) await withFixture({}, async (value, directory) => {
|
||||
const file = path.join(directory, value.sidecar[field]);
|
||||
await writeFile(file, Buffer.concat([await readFile(file), Buffer.from(' ')]));
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /bytes changed/);
|
||||
});
|
||||
});
|
||||
|
||||
test('source, compiled build and owning report must match the arm', async () => {
|
||||
const cases = [
|
||||
[{ sidecar: row => { row.sourceRevision = '3'.repeat(40); } }, /source revision differs/],
|
||||
[{ sidecar: row => { row.compiledBuildRevision = '3'.repeat(40); } }, /build revision differs/],
|
||||
[{ sidecar: row => { row.playwrightReportSha256 = hash('another-report'); } }, /another owning Playwright/],
|
||||
[{ manifest: row => { row.revision = '3'.repeat(40); } }, /manifest source revision/],
|
||||
[{ receipt: row => { row.results[0].revision = '3'.repeat(40); } }, /receipt source revision/],
|
||||
[{ receipt: row => { row.results[0].compiledBuildRevision = '3'.repeat(40); } }, /receipt build revision/],
|
||||
[{ receipt: row => { row.results[0].sourceClean = false; } }, /source was not clean/],
|
||||
];
|
||||
for (const [options, message] of cases) await withFixture(options, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('file descriptors must be complete, bounded, distinct and match the Python aggregate', async () => {
|
||||
const cases = [
|
||||
[row => { row.fileCount--; }, /file count is incomplete/],
|
||||
[row => { row.aggregateSha256 = hash('wrong aggregate'); }, /aggregate does not match/],
|
||||
[row => { row.files[0].bytes = -1; }, /file descriptor/],
|
||||
[row => { row.files[0].path = '../outside'; }, /escapes registered roots/],
|
||||
[row => { row.files[1].path = row.files[0].path; }, /Duplicate/],
|
||||
[row => { row.files.reverse(); }, /order differs/],
|
||||
[row => { row.files.pop(); row.fileCount--; row.aggregateSha256 = aggregate(row.files); }, /omitted a registered root/],
|
||||
[row => { row.roots = ['server/dist']; }, /registered inventory roots/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ manifest: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('summary identifies exact gzip, count and aggregate for the selected variant', async () => {
|
||||
for (const [key, value] of [['manifestSha256', hash('wrong')], ['fileCount', 1], ['aggregateSha256', hash('wrong')]]) {
|
||||
await withFixture({ summary: row => { row.inventories.B2[key] = value; } }, async fixture => {
|
||||
await assert.rejects(validateArmInventory(fixture.sidecarFile, fixture), /Inventory summary/);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('serving entry, every hot path and native binary must match full evidence', async () => {
|
||||
const cases = [
|
||||
[row => { row.serverEntrySha256 = hash('wrong-entry'); }, /server entry differs/],
|
||||
[row => { row.compiledHotPathHashes['server/dist/hot-c.js'] = hash('wrong-hot-path'); }, /hot path differs/],
|
||||
[row => { row.remote.runner.sha256 = hash('wrong-native'); }, /native binary differs/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ evidence: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
await withFixture({ receipt: row => { row.nativeStageHashes.push({ ...row.nativeStageHashes[0] }); } }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /native stage exactly once/);
|
||||
});
|
||||
});
|
||||
|
||||
test('a future candidate revision is accepted only with matching complete provenance', async () => {
|
||||
await withFixture({ revision: 'f'.repeat(40) }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.revision, 'f'.repeat(40));
|
||||
assert.equal(proof.compiledBuildRevision, 'f'.repeat(40));
|
||||
});
|
||||
});
|
||||
|
||||
test('aggregate follows the existing Python ensure_ascii convention for non-ASCII paths', async () => {
|
||||
// Independently computed with the retained artifact-inventory-ce33.py
|
||||
// json.dumps(sort_keys=True, separators=(',', ':')) canonical convention.
|
||||
const expected = '6f7bf8cab07e2aaf8faaa95a8bdb565c588670e4e3e3580acdba1f93154fedde';
|
||||
await withFixture({ manifest: row => {
|
||||
row.files.at(-1).path = 'ui/dist/é😀\u007f.html'; row.aggregateSha256 = expected;
|
||||
} }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.aggregateSha256, expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { createRequire } from 'node:module';
|
||||
import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate,
|
||||
H5_LIMITS, ORIGINAL_BUDGETS } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
|
||||
const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`),
|
||||
{ matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } });
|
||||
|
||||
test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
assert.equal(require('@playwright/test/package.json').version, '1.62.1');
|
||||
await assert.rejects(expect.poll(async () => 0, { timeout: 15, intervals: [5] }).toBeGreaterThanOrEqual(1), error => {
|
||||
assert.equal(error.matcherResult?.name, 'toBeGreaterThanOrEqual');
|
||||
assert.equal(error.matcherResult.pass, false);
|
||||
assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate'));
|
||||
return true;
|
||||
});
|
||||
// This checks the real library's format; it does not execute a120s wait or
|
||||
// claim runtime qualification of H5's extended application path.
|
||||
});
|
||||
|
||||
test('H5 requires explicit mode and leaves default H4c selected', () => {
|
||||
assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true);
|
||||
assert.throws(() => h5Enabled('almost-h5'));
|
||||
});
|
||||
|
||||
test('first successful observation uses only the original wait with no collection delay', async () => {
|
||||
const calls = [];
|
||||
const value = await observeStreaming({ readCount: async () => 16, target: 16, now: () => 1000,
|
||||
poll: async (read, milliseconds, target) => { calls.push(milliseconds); assert.ok(await read() >= target); } });
|
||||
assert.deepEqual(calls, [120000]); assert.equal(value.original.status, 'passed');
|
||||
assert.equal(value.observation.complete, true); assert.equal(value.observation.completedAtMs, 1000);
|
||||
});
|
||||
|
||||
test('known120s miss remains failed while the same observation finishes before600s', async () => {
|
||||
let clock = 1000, count = 30; const calls = [], records = [];
|
||||
const value = await observeStreaming({ readCount: async () => count, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => {
|
||||
calls.push(milliseconds); await read();
|
||||
if (calls.length === 1) { clock += milliseconds; throw timeout(milliseconds); }
|
||||
count = 512; clock += 90000; await read();
|
||||
} });
|
||||
assert.deepEqual(calls, [120000, 480000]);
|
||||
assert.equal(value.original.status, 'failed'); assert.equal(value.original.lastObservedCount, 30);
|
||||
assert.equal(value.original.lastObservedAtMs, 1000); assert.equal(value.original.deadlineMs, 121000);
|
||||
assert.equal(value.observation.completedAtMs, 211000); assert.equal(records[0].observation.complete, false);
|
||||
assert.equal(value.original.reason, 'original_streaming_deadline');
|
||||
});
|
||||
|
||||
test('page read failures and unknown poll failures never become an expected SLA miss', async () => {
|
||||
const failure = new Error('page detached');
|
||||
await assert.rejects(observeStreaming({ readCount: async () => { throw failure; }, target: 1,
|
||||
poll: async read => { try { await read(); } catch {} throw timeout(120000); } }), error => error === failure);
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 0, target: 1,
|
||||
poll: async () => { throw new Error('unrelated assertion'); } }), /Only the original streaming matcher deadline/);
|
||||
});
|
||||
|
||||
test('observation ceiling leaves a censored failure, never a substituted latency', async () => {
|
||||
let clock = 1000, calls = 0; const records = [];
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 30, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => { await read(); calls++; clock += milliseconds; throw timeout(milliseconds); } }), /Timeout/);
|
||||
assert.equal(calls, 2); assert.equal(clock, 601000);
|
||||
assert.equal(records.at(-1).observation.complete, false); assert.equal(records.at(-1).observation.censored, true);
|
||||
assert.equal(records.at(-1).observation.completedAtMs, null);
|
||||
});
|
||||
|
||||
test('original terminal is not reached after120s miss; observational180s is distinct', () => {
|
||||
const terminal = { complete: true, startedAtMs: 211000, deadlineMs: 391000, assessedAtMs: 212000 };
|
||||
assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), {
|
||||
status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null });
|
||||
assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed');
|
||||
assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
});
|
||||
|
||||
test('cursor inherits only paired exact known misses; final candidate inherits none', () => {
|
||||
const base = { misses: ['burst512:streaming120'] };
|
||||
requireH5Candidate('cursor-nonregression', base, base);
|
||||
const slowInvocation = { misses: ['invocation:observed-total-duration900'] };
|
||||
requireH5Candidate('cursor-nonregression', slowInvocation, slowInvocation);
|
||||
requireH5Candidate('final-b2', base, { misses: [] });
|
||||
for (const candidate of [{ misses: ['paced:streaming120'] }, { misses: ['invocation:observed-total-duration900'] },
|
||||
{ misses: ['operator-stop:streaming120'] }]) assert.throws(() => requireH5Candidate('cursor-nonregression', base, candidate));
|
||||
assert.throws(() => requireH5Candidate('final-b2', base, base), /every original SLA/);
|
||||
assert.throws(() => requireH5Candidate('final-b2', slowInvocation, slowInvocation), /every original SLA/);
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { H5_MODE, H5_PROTOCOL, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, originalTerminal } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
|
||||
test("four pairs must individually improve beyond noise", () => {
|
||||
@@ -161,6 +163,196 @@ function browserReport(revision, latency) {
|
||||
}) };
|
||||
}
|
||||
|
||||
function h5Report(revision, loadedLatency) {
|
||||
const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000);
|
||||
Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE,
|
||||
workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true,
|
||||
actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture",
|
||||
serverEntrySha256: "e".repeat(64), compiledHotPathHashes: {
|
||||
"server/dist/services/heartbeat-run-events.js": "f".repeat(64),
|
||||
"server/dist/services/native-runtime/paperclip-control-plane-port.js": "1".repeat(64),
|
||||
"server/dist/vendor/paperclip-runner/native-session-runtime.js": "2".repeat(64) } });
|
||||
for (const item of report.cases) {
|
||||
const latency = item.name === "burst512" ? loadedLatency : 1000;
|
||||
item.settledAtMs = item.epochMs + latency; item.drainThroughTerminalMs = latency;
|
||||
item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency };
|
||||
item.intendedToVisible = { p95Ms: latency };
|
||||
if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1;
|
||||
item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs }));
|
||||
const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs;
|
||||
const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null,
|
||||
startedAtMs: item.epochMs, deadlineMs: item.epochMs + 120000, assessedAtMs,
|
||||
target: item.stop ? 3 : item.events, lastObservedCount: missed ? 30 : item.visible.length,
|
||||
lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs,
|
||||
error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null },
|
||||
observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } };
|
||||
const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000,
|
||||
assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs,
|
||||
lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false };
|
||||
item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation),
|
||||
visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length,
|
||||
independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10,
|
||||
deltaCount: item.runs * (item.stop ? 3 : item.events), resultAcceptedCount: item.stop ? 0 : item.runs, clockStreamsClosed: true } };
|
||||
item.terminalStatuses = item.runIds.map(runId => ({ runId, runtimeMode: "native", status: item.stop ? "cancelled" : "succeeded" }));
|
||||
for (const row of item.durable) row.resultAcceptedEvents = item.stop ? 0 : 1;
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires complete owning evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; };
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" });
|
||||
const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map();
|
||||
async function inventory(report, playwrightFile, name) {
|
||||
const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision];
|
||||
if (!inventories.has(variant)) {
|
||||
const roots = ["server/dist", "ui/dist", "packages/db/src", "packages/shared/src"];
|
||||
const files = Object.entries({ ...report.compiledHotPathHashes, "server/dist/index.js": report.serverEntrySha256,
|
||||
"packages/db/src/client.ts": "3".repeat(64), "packages/shared/src/index.ts": "4".repeat(64), "ui/dist/index.html": "5".repeat(64) })
|
||||
.map(([path, sha256]) => ({ bytes: 10, path, sha256 })).sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
const aggregateSha256 = hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join("\n") + "\n");
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify({ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, roots, fileCount: files.length, files, aggregateSha256 })));
|
||||
const manifestFile = path.join(directory, `${variant}-artifact-inventory.json.gz`); await writeFile(manifestFile, manifestBytes);
|
||||
const manifestFileSha256 = hash(manifestBytes);
|
||||
const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: {
|
||||
manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } });
|
||||
const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40),
|
||||
binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }],
|
||||
artifactInventorySummary: inventorySummaryFile });
|
||||
inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile,
|
||||
inventorySummarySha256: hash(await readFile(inventorySummaryFile)), qualifiedReceiptFile,
|
||||
qualifiedReceiptSha256: hash(await readFile(qualifiedReceiptFile)) });
|
||||
}
|
||||
return save(`${name}-inventory.json`, { schema: "paperclip.native-ingestion.arm-inventory.v1", variant,
|
||||
sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision,
|
||||
playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) });
|
||||
}
|
||||
async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) {
|
||||
const entries = [];
|
||||
for (let index = 0; index < 5; index++) {
|
||||
const entry = {};
|
||||
for (const arm of ["baseline", "candidate"]) {
|
||||
const report = h5Report(arm === "baseline" || sameRevision ? baselineRevision : candidateRevision, arm === "baseline" ? 180000 : candidateLatency);
|
||||
const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0,
|
||||
duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] };
|
||||
if (index === 0 && arm === "candidate") mutate?.(report, owner);
|
||||
owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json",
|
||||
body: Buffer.from(JSON.stringify(report)).toString("base64") });
|
||||
entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report);
|
||||
entry[`${arm}PlaywrightFile`] = await save(`${prefix}-${index}-${arm}-playwright.json`, { suites: [{ tests: [owner] }] });
|
||||
entry[`${arm}InventoryFile`] = await inventory(report, entry[`${arm}PlaywrightFile`], `${prefix}-${index}-${arm}`);
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
return save(`${prefix}-pairs.json`, entries);
|
||||
}
|
||||
try {
|
||||
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
|
||||
proof.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream";
|
||||
proof.measurementSha256 = "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b";
|
||||
proof.harnessProvenance = { sourceRevision: "7d7e7a88d562a95a1d407b76474763b666c181f5", sourceStatus: "" };
|
||||
for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field];
|
||||
for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; }
|
||||
const originalProof = await save("original-h4c-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{
|
||||
status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{
|
||||
name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] });
|
||||
const originalInventory = await inventory(proof, originalProof, "original-h4c");
|
||||
const finalProofArgs = [`--original-h4c-proof=${originalProof}`, `--original-h4c-inventory=${originalInventory}`];
|
||||
const noise = path.join(directory, "aa.json");
|
||||
const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]);
|
||||
assert.equal(aa.status, 0, aa.stderr);
|
||||
const aaReport = JSON.parse(await readFile(noise, "utf8"));
|
||||
assert.equal(aaReport.h5Assessments.length, 5);
|
||||
assert.deepEqual(aaReport.h5Assessments[0].baseline.misses, ["burst512:streaming120"]);
|
||||
const cursor = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor", 180000, { candidateRevision: "B1" })}`, `--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor.status, 0, cursor.stderr);
|
||||
const cursor900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-matched900", 180000, { candidateRevision: "B1", duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor900.status, 0, cursor900.stderr);
|
||||
const added900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-added900", 180000, { candidateRevision: "B1", mutate: (_report, owner) => { owner.results[0].duration = 950000; } })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(added900.status, 0, "Cursor candidate cannot add an unmatched900 miss");
|
||||
const goodPairs = await manifest("good", 1000);
|
||||
const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(good.status, 0, good.stderr);
|
||||
const wrongPolicy = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", `--pairs=${goodPairs}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(wrongPolicy.status, 0); assert.match(wrongPolicy.stderr, /Cursor policy requires B0→B1/);
|
||||
const b1Noise = path.join(directory, "aa-b1.json");
|
||||
const b1AA = run(["--phase=aa", `--pairs=${await manifest("aa-b1", 181000, { sameRevision: true, baselineRevision: "B1" })}`, `--output=${b1Noise}`]);
|
||||
assert.equal(b1AA.status, 0, b1AA.stderr);
|
||||
const incremental = run(["--candidate-policy=final-b2", `--pairs=${await manifest("b1-b2", 1000, { baselineRevision: "B1" })}`,
|
||||
`--noise-report=${b1Noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(incremental.status, 0, incremental.stderr);
|
||||
const pendingFile = path.join(directory, "pending-h4c.json");
|
||||
const pending = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, `--output=${pendingFile}`, "--require-acceptance"]);
|
||||
assert.notEqual(pending.status, 0);
|
||||
const pendingReport = JSON.parse(await readFile(pendingFile, "utf8"));
|
||||
assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false);
|
||||
const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.equal(performanceOnly.status, 0, performanceOnly.stderr);
|
||||
for (const [name, change, message] of [
|
||||
["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/],
|
||||
["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/],
|
||||
["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /unchanged original H4c/],
|
||||
]) {
|
||||
const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0],
|
||||
value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString());
|
||||
change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64");
|
||||
const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`);
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
|
||||
`--original-h4c-proof=${file}`, `--original-h4c-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]);
|
||||
assert.notEqual(attempt.status, 0, name);
|
||||
const rejected = JSON.parse(await readFile(output, "utf8"));
|
||||
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
|
||||
assert.match(rejected.originalH4cConfirmation.reason, message);
|
||||
}
|
||||
const badOutput = path.join(directory, "still-misses.json");
|
||||
const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`,
|
||||
`--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]);
|
||||
assert.notEqual(bad.status, 0);
|
||||
const rejected = JSON.parse(await readFile(badOutput, "utf8"));
|
||||
assert.equal(rejected.acceptancePassed, false);
|
||||
assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false);
|
||||
for (const [name, mutate, message] of [
|
||||
["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/],
|
||||
["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/],
|
||||
["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/],
|
||||
["missing-delta", report => { report.cases[3].durable[99].individualDeltas--; }, /individualDeltas|result.events|512/],
|
||||
["wrong-workload", report => { report.workloadSha256 = "old"; }, /registration changed/],
|
||||
["pw-retry", (_report, owner) => { owner.results[0].retry = 1; }, /Retries/],
|
||||
["pw-error", (_report, owner) => { owner.results[0].errors.push({ message: "unexpected API failure" }); }, /deep-equal/],
|
||||
["wrong-observation-timeout", (_report, owner) => { owner.timeout = 900000; }, /1200000/],
|
||||
]) {
|
||||
const result = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutate })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(result.status, 0, name); assert.match(result.stderr, message, name);
|
||||
}
|
||||
const outer = run(["--candidate-policy=final-b2", `--pairs=${await manifest("outer900", 1000, { duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(outer.status, 0, "Observed total duration900 miss cannot qualify final B2");
|
||||
const incomplete = JSON.parse(await readFile(await manifest("missing-owner", 1000), "utf8"));
|
||||
delete incomplete[0].candidatePlaywrightFile;
|
||||
const missing = run(["--candidate-policy=final-b2", `--pairs=${await save("missing-owner-manifest.json", incomplete)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(missing.status, 0); assert.match(missing.stderr, /independent owning Playwright/);
|
||||
const global = JSON.parse(await readFile(await manifest("global-error", 1000), "utf8"));
|
||||
const globalOwner = JSON.parse(await readFile(global[0].candidatePlaywrightFile, "utf8"));
|
||||
globalOwner.errors = [{ message: "global teardown failed" }];
|
||||
await writeFile(global[0].candidatePlaywrightFile, JSON.stringify(globalOwner));
|
||||
const globalFailed = run(["--candidate-policy=final-b2", `--pairs=${await save("global-error-manifest.json", global)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(globalFailed.status, 0); assert.match(globalFailed.stderr, /global error/);
|
||||
} finally { await rm(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
|
||||
Reference in new issue
Block a user