mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: observe full ingestion drainage with original deadline outcomes
This commit is contained in:
Dotta committed
1 parent
7d7e7a88d5
commit
81c9ffdf03
10 files changed
+1120
-14
No files matched your search
@@ -3,6 +3,8 @@ import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
import { H5_MODE, H5_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs";
|
||||
|
||||
// Each manifest entry points at native-ingestion-evidence JSON attachments
|
||||
// from two isolated compiled-app runs. Run the variants in alternating order;
|
||||
@@ -11,12 +13,87 @@ const arg = (name: string) => process.argv.find(value => value.startsWith(`--${n
|
||||
const manifestPath = arg("pairs");
|
||||
assert.ok(manifestPath, "Pass --pairs=<JSON manifest of baselineFile/candidateFile pairs>");
|
||||
const phase = arg("phase") ?? "compare", expectation = arg("expect") ?? "improvement";
|
||||
const h5 = arg("observation-mode") === H5_MODE;
|
||||
assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode");
|
||||
const candidatePolicy = arg("candidate-policy");
|
||||
assert.ok(["aa", "compare"].includes(phase));
|
||||
assert.ok(["improvement", "nonregression"].includes(expectation));
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string }>;
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string;
|
||||
baselinePlaywrightFile?: string; candidatePlaywrightFile?: string; baselineInventoryFile?: string; candidateInventoryFile?: string }>;
|
||||
assert.ok(manifest.length >= 5, "At least five actual paired application runs are required");
|
||||
if (h5) assert.equal(manifest.length, 5, "H5 requires exactly five prospectively registered pairs");
|
||||
const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8"));
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) })));
|
||||
async function owningResult(reportPath: string, evidence?: any) {
|
||||
const bytes = await readFile(reportPath);
|
||||
const report = JSON.parse(bytes.toString("utf8")), tests: any[] = [];
|
||||
assert.deepEqual(report.errors ?? [], [], "Owning Playwright report has a global error");
|
||||
const visit = (value: any) => {
|
||||
if (Array.isArray(value)) for (const child of value) visit(child);
|
||||
else if (value && typeof value === "object") {
|
||||
if (Array.isArray(value.results) && Object.hasOwn(value, "expectedStatus")) tests.push(value);
|
||||
for (const [key, child] of Object.entries(value)) if (key !== "results") visit(child);
|
||||
}
|
||||
};
|
||||
visit(report); assert.equal(tests.length, 1, "One complete owning test is required; no partial selection");
|
||||
const test = tests[0]; assert.equal(test.results.length, 1, "No retry or omitted owning attempt");
|
||||
assert.equal(test.expectedStatus, "passed");
|
||||
const result = test.results[0], attachments = result.attachments.filter((item: any) => item.name === "native-ingestion-evidence");
|
||||
assert.ok(result.error === undefined || result.error === null, "Owning Playwright attempt has an error");
|
||||
assert.equal(attachments.length, 1, "Owning result must carry this exact evidence attachment");
|
||||
const attachment = attachments[0];
|
||||
const attached = attachment.body ? JSON.parse(Buffer.from(attachment.body, "base64").toString("utf8"))
|
||||
: JSON.parse(await readFile(path.resolve(path.dirname(reportPath), attachment.path), "utf8"));
|
||||
if (evidence) assert.deepEqual(attached, evidence, "Playwright result belongs to a different evidence arm");
|
||||
return { evidence: attached, owner: { reportFile: reportPath, reportSha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
status: result.status, retry: result.retry, errors: result.errors, startedAt: result.startTime,
|
||||
durationMs: result.duration, timeoutMs: test.timeout } };
|
||||
}
|
||||
async function loadArm(file: string, playwrightFile?: string, inventoryFile?: string) {
|
||||
const evidence = await load(file);
|
||||
if (!h5) return evidence;
|
||||
assert.ok(playwrightFile, "H5 requires each arm's independent owning Playwright JSON report");
|
||||
evidence.owningPlaywright = (await owningResult(path.resolve(path.dirname(manifestPath!), playwrightFile), evidence)).owner;
|
||||
assert.ok(inventoryFile, "H5 requires each arm's exact full artifact inventory sidecar");
|
||||
evidence.armInventory = await validateArmInventory(path.resolve(path.dirname(manifestPath!), inventoryFile),
|
||||
{ evidence, owningPlaywright: evidence.owningPlaywright });
|
||||
return evidence;
|
||||
}
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await loadArm(entry.baselineFile, entry.baselinePlaywrightFile, entry.baselineInventoryFile),
|
||||
candidate: await loadArm(entry.candidateFile, entry.candidatePlaywrightFile, entry.candidateInventoryFile) })));
|
||||
const h5Assessments: any[] = [];
|
||||
if (h5) {
|
||||
if (phase === "compare") {
|
||||
assert.ok(["final-b2", "cursor-nonregression"].includes(candidatePolicy ?? ""), "H5 requires an explicit candidate SLA policy");
|
||||
assert.equal(expectation, candidatePolicy === "cursor-nonregression" ? "nonregression" : "improvement");
|
||||
}
|
||||
for (const pair of pairs) {
|
||||
assert.ok(Object.values(H5_BASELINE_REVISIONS).includes(pair.baseline.sourceRevision), "H5 baseline must use the prospectively frozen B0 or B1 revision");
|
||||
const baselineVariant = pair.baseline.sourceRevision === H5_BASELINE_REVISIONS.B0 ? "B0" : "B1";
|
||||
assert.equal(pair.baseline.armInventory.variant, baselineVariant, "Baseline inventory role differs from its frozen revision");
|
||||
if (phase === "aa") assert.equal(pair.candidate.armInventory.variant, baselineVariant, "A/A must use the same baseline inventory role");
|
||||
if (phase === "compare" && candidatePolicy === "cursor-nonregression") {
|
||||
assert.equal(pair.baseline.sourceRevision, H5_BASELINE_REVISIONS.B0, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.sourceRevision, H5_BASELINE_REVISIONS.B1, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.armInventory.variant, "B1", "Cursor policy cannot stand in for final B2 acceptance");
|
||||
} else if (phase === "compare") {
|
||||
assert.equal(pair.candidate.armInventory.variant, "B2", "Final policy requires the qualified B2 inventory role");
|
||||
assert.ok(!Object.values(H5_BASELINE_REVISIONS).includes(pair.candidate.sourceRevision), "Final B2 must differ from both frozen baselines");
|
||||
}
|
||||
const baseline = assessH5Arm(pair.baseline, pair.baseline.owningPlaywright);
|
||||
const candidate = assessH5Arm(pair.candidate, pair.candidate.owningPlaywright);
|
||||
// The only prospectively known baseline workload SLA miss is loaded streaming.
|
||||
const allowed = new Set(["burst512:streaming120", "invocation:observed-total-duration900"]);
|
||||
for (const miss of baseline.misses) assert.ok(allowed.has(miss), "Unknown baseline deadline failure needs diagnosis");
|
||||
if (phase === "aa") for (const miss of candidate.misses) assert.ok(allowed.has(miss), "Unknown A/A deadline failure needs diagnosis");
|
||||
let candidateSlaPolicyError;
|
||||
if (phase === "compare") {
|
||||
try { requireH5Candidate(candidatePolicy, baseline, candidate); }
|
||||
catch (error) { candidateSlaPolicyError = error instanceof Error ? error.message : String(error); }
|
||||
}
|
||||
h5Assessments.push({ baseline, candidate, candidateSlaPolicyError, candidateSlaPolicyPassed: !candidateSlaPolicyError,
|
||||
owningPlaywright: { baseline: pair.baseline.owningPlaywright, candidate: pair.candidate.owningPlaywright } });
|
||||
}
|
||||
}
|
||||
const workloads = [
|
||||
{ name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
@@ -32,7 +109,8 @@ const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/
|
||||
const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
|
||||
for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
|
||||
const report = pair[arm];
|
||||
assert.equal(report.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream", "Fresh browser acceptance requires the qualified v4 mixed clock protocol");
|
||||
assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : "v4-remote-streaming-original-local-stop-clock-stream",
|
||||
h5 ? "Fresh browser acceptance requires its qualified H5 observation protocol" : "Fresh browser acceptance requires the qualified v4 mixed clock protocol");
|
||||
assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport");
|
||||
assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, "Missing explicit transport and real cloud Stop qualification policy");
|
||||
@@ -97,6 +175,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm");
|
||||
assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples");
|
||||
assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples");
|
||||
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples");
|
||||
assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs");
|
||||
if (phase === "aa") assert.equal(report.sourceRevision, first.baseline.sourceRevision, "A/A must use one source revision");
|
||||
for (const name of names) {
|
||||
@@ -300,6 +380,56 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`);
|
||||
}
|
||||
}
|
||||
let originalH4cConfirmation: Record<string, any> | undefined;
|
||||
if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
originalH4cConfirmation = { required: true, passed: false, reason: "Later unchanged original H4c full functional proof is pending" };
|
||||
if (arg("original-h4c-proof")) {
|
||||
try {
|
||||
assert.ok(arg("original-h4c-inventory"), "Original H4c proof requires its full artifact inventory sidecar");
|
||||
const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4c-proof")!));
|
||||
assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.equal(proof.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream");
|
||||
assert.equal(proof.observationMode, undefined, "Confirmation must use unchanged original H4c defaults");
|
||||
assert.equal(proof.harnessProvenance?.sourceRevision, "7d7e7a88d562a95a1d407b76474763b666c181f5");
|
||||
assert.equal(proof.harnessProvenance.sourceStatus, "");
|
||||
assert.equal(proof.measurementSha256, "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b");
|
||||
assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false);
|
||||
assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled");
|
||||
assert.equal(proof.cleanup?.complete, true); assert.deepEqual(proof.cleanup.errors, []);
|
||||
assert.deepEqual(proof.browserRecording, first.candidate.browserRecording);
|
||||
assert.equal(proof.sourceRevision, first.candidate.sourceRevision); assert.equal(proof.compiledBuildRevision, first.candidate.compiledBuildRevision);
|
||||
assert.equal(proof.serverEntrySha256, first.candidate.serverEntrySha256);
|
||||
assert.deepEqual(proof.compiledHotPathHashes, first.candidate.compiledHotPathHashes);
|
||||
assert.equal(proof.transport, first.candidate.transport); assert.deepEqual(proof.transportPolicy, first.candidate.transportPolicy);
|
||||
assert.deepEqual(proof.cases.map((item: any) => item.name), names, "Original confirmation must include all six workloads");
|
||||
for (const fixed of workloads) {
|
||||
const item = proof.cases.find((item: any) => item.name === fixed.name);
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(item[field], fixed[field]);
|
||||
assert.equal(item.persistedAfterReload, true); assert.equal(item.durable.length, fixed.runs);
|
||||
assert.equal(item.matchedVisibleClocks, item.visible.length);
|
||||
assert.ok(item.visible.length >= (fixed.stop ? 3 : fixed.events));
|
||||
assert.equal(item.runIds.length, fixed.runs); assert.equal(new Set(item.runIds).size, fixed.runs);
|
||||
for (const row of item.durable) {
|
||||
assert.equal(row.ordered, true); assert.equal(row.uniqueSourceIdentities, row.individualDeltas);
|
||||
assert.equal(row.matchedSourceClocks, row.individualDeltas);
|
||||
if (!fixed.stop) assert.equal(row.individualDeltas, fixed.events);
|
||||
}
|
||||
}
|
||||
const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs));
|
||||
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
|
||||
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4c-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
|
||||
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
|
||||
originalH4cConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision };
|
||||
} catch (error) {
|
||||
originalH4cConfirmation.reason = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
const metric = (item: any, key: string): number => key.split(".").reduce((value, part) => value[part], item);
|
||||
const measurements = names.flatMap(name => {
|
||||
const observed = first.baseline.cases.find((item: any) => item.name === name);
|
||||
@@ -334,11 +464,21 @@ const result: Record<string, any> = {
|
||||
comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
|
||||
manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements,
|
||||
primaryLoadedWorkload: "burst512", complete: true,
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments,
|
||||
artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4cConfirmation,
|
||||
originalProtocolAcceptance: "Baseline SLA misses are retained; H5 complete observation is not H4c PASS",
|
||||
finalUnchangedH4cFunctionalPassRequired: candidatePolicy === "final-b2" } : {}),
|
||||
};
|
||||
if (phase === "compare") {
|
||||
assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report");
|
||||
const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8"));
|
||||
assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true);
|
||||
if (h5) {
|
||||
assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256);
|
||||
assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required");
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A");
|
||||
}
|
||||
assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol");
|
||||
assert.equal(noise.transport, result.transport, "Noise report uses a different transport");
|
||||
assert.deepEqual(noise.transportPolicy, result.transportPolicy, "Transport policy changed after A/A calibration");
|
||||
@@ -362,10 +502,17 @@ if (phase === "compare") {
|
||||
noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
|
||||
passed: improvementRequired ? improvement : noRegression };
|
||||
});
|
||||
result.acceptancePassed = result.gates.every((entry: any) => entry.passed);
|
||||
result.performancePassed = result.gates.every((entry: any) => entry.passed)
|
||||
&& (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed));
|
||||
result.acceptancePassed = result.performancePassed && (!originalH4cConfirmation || originalH4cConfirmation.passed);
|
||||
}
|
||||
if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (process.argv.includes("--require-performance")) {
|
||||
assert.equal(h5, true, "H5 performance-only gating is explicit; original H4c remains unchanged");
|
||||
assert.equal(phase, "compare");
|
||||
assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose");
|
||||
}
|
||||
if (process.argv.includes("--require-acceptance")) {
|
||||
assert.equal(phase, "compare", "A/A calibrates noise; it is not performance acceptance");
|
||||
assert.equal(result.acceptancePassed, true, "Browser performance acceptance failed; retain results and diagnose");
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { H5_REGISTRATION, H5_WORKLOAD_SHA256 } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
if (process.argv.length > 2) throw new Error('Dry run only: this command accepts no execution arguments');
|
||||
console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadSha256: H5_WORKLOAD_SHA256,
|
||||
registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes',
|
||||
'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation',
|
||||
'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass',
|
||||
'Later unchanged original H4c full functional pass on the final SUT'] }, null, 2));
|
||||
@@ -0,0 +1,137 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
const SCHEMA = 'paperclip.native-ingestion.arm-inventory.v1';
|
||||
const ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const sha256 = value => createHash('sha256').update(value).digest('hex');
|
||||
const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
||||
const revision = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
|
||||
const object = value => value && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
async function bounded(file, maximumBytes) {
|
||||
const metadata = await stat(file);
|
||||
assert.ok(metadata.isFile() && metadata.size <= maximumBytes, 'Inventory evidence file is absent, nonregular or oversized');
|
||||
const bytes = await readFile(file);
|
||||
assert.ok(bytes.length <= maximumBytes, 'Inventory evidence file grew beyond its bound');
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function relativeArtifact(value) {
|
||||
return typeof value === 'string' && value.length > 0 && !value.includes('\\')
|
||||
&& !value.includes('\0') && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value
|
||||
&& !value.split('/').includes('..') && ROOTS.some(root => value.startsWith(`${root}/`));
|
||||
}
|
||||
|
||||
// The existing Python generator sorts Unicode code points and uses json.dumps
|
||||
// with sort_keys=True, separators=(',', ':') and its default ensure_ascii=True.
|
||||
function pythonOrder(left, right) {
|
||||
const a = [...left], b = [...right];
|
||||
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
||||
const difference = a[index].codePointAt(0) - b[index].codePointAt(0);
|
||||
if (difference) return difference;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })
|
||||
.replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
||||
|
||||
/**
|
||||
* Validates frozen inventory evidence and reported serving fingerprints only.
|
||||
* The owner supplies this sidecar per arm; this helper never reads the SUT tree
|
||||
* and does not attest that a live filesystem remained unchanged during a run.
|
||||
*/
|
||||
export async function validateArmInventory(sidecarFile, { evidence, owningPlaywright }) {
|
||||
assert.equal(typeof sidecarFile, 'string', 'Missing arm inventory sidecar');
|
||||
assert.ok(object(evidence) && object(owningPlaywright), 'Missing serving evidence or owning Playwright report');
|
||||
const absoluteSidecar = path.resolve(sidecarFile), base = path.dirname(absoluteSidecar);
|
||||
const sidecarBytes = await bounded(absoluteSidecar, 256 * 1024);
|
||||
const sidecar = JSON.parse(sidecarBytes.toString('utf8'));
|
||||
assert.equal(sidecar.schema, SCHEMA, 'Unsupported arm inventory sidecar schema');
|
||||
assert.ok(['B0', 'B1', 'B2'].includes(sidecar.variant), 'Unknown inventory variant');
|
||||
assert.ok(revision(sidecar.sourceRevision) && revision(sidecar.compiledBuildRevision), 'Invalid inventory source revision');
|
||||
assert.equal(sidecar.sourceRevision, evidence.sourceRevision, 'Inventory source revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, evidence.compiledBuildRevision, 'Inventory build revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, sidecar.sourceRevision, 'Inventory was built from a different source revision');
|
||||
assert.ok(digest(sidecar.playwrightReportSha256) && digest(owningPlaywright.reportSha256), 'Missing owning Playwright report hash');
|
||||
assert.equal(sidecar.playwrightReportSha256, owningPlaywright.reportSha256, 'Inventory belongs to another owning Playwright report');
|
||||
const resolve = name => {
|
||||
assert.ok(typeof sidecar[name] === 'string' && sidecar[name].length > 0, `Missing inventory ${name}`);
|
||||
return path.resolve(base, sidecar[name]);
|
||||
};
|
||||
const readHashed = async (name, hashName, maximumBytes) => {
|
||||
assert.ok(digest(sidecar[hashName]), `Invalid inventory ${hashName}`);
|
||||
const file = resolve(name), bytes = await bounded(file, maximumBytes);
|
||||
assert.equal(sha256(bytes), sidecar[hashName], `Inventory ${name} bytes changed`);
|
||||
return { file, bytes };
|
||||
};
|
||||
const manifestSource = await readHashed('manifestFile', 'manifestFileSha256', 64 * 1024 * 1024);
|
||||
const manifest = JSON.parse(gunzipSync(manifestSource.bytes, { maxOutputLength: 128 * 1024 * 1024 }).toString('utf8'));
|
||||
assert.deepEqual(Object.keys(manifest).sort(), ['aggregateSha256', 'compiledBuildRevision', 'fileCount', 'files', 'revision', 'roots', 'variant']);
|
||||
assert.equal(manifest.variant, sidecar.variant, 'Inventory manifest variant differs from sidecar');
|
||||
assert.equal(manifest.revision, sidecar.sourceRevision, 'Inventory manifest source revision differs from arm');
|
||||
assert.equal(manifest.compiledBuildRevision, sidecar.compiledBuildRevision, 'Inventory manifest build revision differs from arm');
|
||||
assert.deepEqual(manifest.roots, ROOTS, 'Full registered inventory roots are required');
|
||||
assert.ok(Number.isSafeInteger(manifest.fileCount) && manifest.fileCount > 0 && manifest.fileCount <= 500000, 'Invalid inventory file count');
|
||||
assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount, 'Inventory file count is incomplete');
|
||||
assert.ok(digest(manifest.aggregateSha256) && digest(sidecar.aggregateSha256), 'Invalid inventory aggregate hash');
|
||||
const files = new Map();
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(object(row), 'Invalid inventory file row');
|
||||
assert.deepEqual(Object.keys(row).sort(), ['bytes', 'path', 'sha256']);
|
||||
assert.ok(relativeArtifact(row.path), 'Inventory file escapes registered roots');
|
||||
assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0 && digest(row.sha256), 'Invalid inventory file descriptor');
|
||||
assert.equal(files.has(row.path), false, 'Duplicate inventory file path');
|
||||
files.set(row.path, row);
|
||||
}
|
||||
const sorted = [...manifest.files].sort((left, right) => pythonOrder(left.path, right.path));
|
||||
assert.deepEqual(manifest.files.map(row => row.path), sorted.map(row => row.path), 'Inventory file order differs from generator');
|
||||
for (const root of ROOTS) assert.ok(manifest.files.some(row => row.path.startsWith(`${root}/`)), 'Inventory omitted a registered root');
|
||||
const aggregate = sha256(Buffer.from(`${sorted.map(pythonRow).join('\n')}\n`, 'utf8'));
|
||||
assert.equal(aggregate, manifest.aggregateSha256, 'Inventory aggregate does not match exact file descriptors');
|
||||
assert.equal(aggregate, sidecar.aggregateSha256, 'Inventory aggregate differs from sidecar');
|
||||
assert.ok(digest(evidence.serverEntrySha256), 'Missing serving server entry hash');
|
||||
assert.equal(files.get('server/dist/index.js')?.sha256, evidence.serverEntrySha256, 'Serving server entry differs from full inventory');
|
||||
assert.ok(object(evidence.compiledHotPathHashes) && Object.keys(evidence.compiledHotPathHashes).length > 0, 'Missing serving hot-path fingerprints');
|
||||
for (const [file, hash] of Object.entries(evidence.compiledHotPathHashes)) {
|
||||
assert.ok(relativeArtifact(file) && digest(hash), 'Invalid serving hot-path fingerprint');
|
||||
assert.equal(files.get(file)?.sha256, hash, 'Serving hot path differs from full inventory');
|
||||
}
|
||||
const summarySource = await readHashed('inventorySummaryFile', 'inventorySummarySha256', 1024 * 1024);
|
||||
const summary = JSON.parse(summarySource.bytes.toString('utf8'));
|
||||
const summaryRow = summary.inventories?.[sidecar.variant];
|
||||
assert.ok(object(summaryRow), 'Inventory summary omitted this variant');
|
||||
assert.ok(typeof summaryRow.manifest === 'string' && path.basename(summaryRow.manifest) === summaryRow.manifest
|
||||
&& summaryRow.manifest.endsWith('.json.gz'), 'Inventory summary manifest reference is invalid');
|
||||
assert.equal(summaryRow.manifestSha256, sidecar.manifestFileSha256, 'Inventory summary points to different manifest bytes');
|
||||
assert.equal(summaryRow.aggregateSha256, aggregate, 'Inventory summary aggregate differs');
|
||||
assert.equal(summaryRow.fileCount, manifest.fileCount, 'Inventory summary file count differs');
|
||||
const receiptSource = await readHashed('qualifiedReceiptFile', 'qualifiedReceiptSha256', 4 * 1024 * 1024);
|
||||
const receipt = JSON.parse(receiptSource.bytes.toString('utf8'));
|
||||
assert.ok(Array.isArray(receipt.results), 'Qualified build receipt has no results');
|
||||
const qualified = receipt.results.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(qualified.length, 1, 'Qualified build receipt must identify this variant exactly once');
|
||||
assert.equal(qualified[0].revision, sidecar.sourceRevision, 'Qualified build receipt source revision differs');
|
||||
assert.equal(qualified[0].compiledBuildRevision, sidecar.compiledBuildRevision, 'Qualified build receipt build revision differs');
|
||||
assert.equal(qualified[0].sourceClean, true, 'Qualified build source was not clean');
|
||||
assert.ok(typeof receipt.artifactInventorySummary === 'string' && receipt.artifactInventorySummary.length > 0,
|
||||
'Qualified build receipt omitted its inventory summary reference');
|
||||
assert.ok(revision(receipt.rustSourceTree), 'Qualified build receipt omitted Rust input identity');
|
||||
assert.ok(Array.isArray(receipt.nativeStageHashes), 'Qualified build receipt omitted native stage hashes');
|
||||
const native = receipt.nativeStageHashes.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(native.length, 1, 'Qualified build receipt must identify this native stage exactly once');
|
||||
assert.ok(digest(native[0].binarySha256) && digest(receipt.binarySha256), 'Qualified native binary hash is invalid');
|
||||
assert.equal(native[0].binarySha256, receipt.binarySha256, 'Qualified native binary hashes disagree');
|
||||
assert.equal(native[0].binarySha256, evidence.remote?.runner?.sha256, 'Serving native binary differs from qualified build');
|
||||
return { schema: SCHEMA, sidecarFile: absoluteSidecar, sidecarFileSha256: sha256(sidecarBytes), variant: sidecar.variant,
|
||||
revision: manifest.revision, compiledBuildRevision: manifest.compiledBuildRevision,
|
||||
manifestFile: manifestSource.file, manifestFileSha256: sidecar.manifestFileSha256,
|
||||
aggregateSha256: aggregate, fileCount: manifest.fileCount, roots: manifest.roots,
|
||||
inventorySummaryFile: summarySource.file, inventorySummarySha256: sidecar.inventorySummarySha256,
|
||||
qualifiedReceiptFile: receiptSource.file, qualifiedReceiptSha256: sidecar.qualifiedReceiptSha256,
|
||||
binarySha256: native[0].binarySha256, rustSourceTree: receipt.rustSourceTree,
|
||||
playwrightReportSha256: owningPlaywright.reportSha256,
|
||||
scope: 'Exact frozen manifest/summary/build-receipt bytes for all registered roots, with reported server/hot-path/native fingerprints checked; no live SUT filesystem attestation' };
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import { validateArmInventory } from './native-ingestion-h5-inventory.mjs';
|
||||
|
||||
const hash = value => createHash('sha256').update(value).digest('hex');
|
||||
const sourceRevision = '1'.repeat(40), reportSha256 = hash('owning-playwright'), binarySha256 = hash('runner');
|
||||
const roots = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const descriptors = () => [
|
||||
'packages/db/src/client.ts', 'packages/shared/src/index.ts', 'server/dist/hot-a.js',
|
||||
'server/dist/hot-b.js', 'server/dist/hot-c.js', 'server/dist/index.js', 'ui/dist/index.html',
|
||||
].map(file => ({ bytes: 10, path: file, sha256: hash(file) }));
|
||||
const aggregate = files => hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join('\n') + '\n');
|
||||
|
||||
async function fixture(directory, options = {}) {
|
||||
const revision = options.revision ?? sourceRevision;
|
||||
const files = descriptors(), manifest = { variant: 'B2', revision, compiledBuildRevision: revision,
|
||||
roots, fileCount: files.length, aggregateSha256: aggregate(files), files };
|
||||
options.manifest?.(manifest);
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify(manifest)));
|
||||
const summary = { capturedAt: '2026-11-03T00:00:00.000Z', inventories: { B2: {
|
||||
manifest: 'B2-artifact-inventory.json.gz', manifestSha256: hash(manifestBytes), fileCount: manifest.fileCount,
|
||||
aggregateSha256: manifest.aggregateSha256 } } };
|
||||
options.summary?.(summary);
|
||||
const receipt = { completedAt: '2026-11-03T00:00:01.000Z',
|
||||
results: [{ variant: 'B2', revision, compiledBuildRevision: revision, sourceClean: true }],
|
||||
rustSourceTree: '2'.repeat(40), binarySha256, nativeStageHashes: [{ variant: 'B2', binarySha256 }],
|
||||
artifactInventorySummary: '/owned/original/artifact-inventory-summary.json' };
|
||||
options.receipt?.(receipt);
|
||||
const summaryBytes = Buffer.from(JSON.stringify(summary)), receiptBytes = Buffer.from(JSON.stringify(receipt));
|
||||
const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant: 'B2', sourceRevision: revision,
|
||||
compiledBuildRevision: revision, playwrightReportSha256: reportSha256,
|
||||
manifestFile: 'B2-artifact-inventory.json.gz', manifestFileSha256: hash(manifestBytes),
|
||||
aggregateSha256: manifest.aggregateSha256,
|
||||
qualifiedReceiptFile: 'qualified.json', qualifiedReceiptSha256: hash(receiptBytes),
|
||||
inventorySummaryFile: 'artifact-inventory-summary.json', inventorySummarySha256: hash(summaryBytes) };
|
||||
options.sidecar?.(sidecar);
|
||||
const evidence = { sourceRevision: revision, compiledBuildRevision: revision,
|
||||
serverEntrySha256: hash('server/dist/index.js'), remote: { runner: { sha256: binarySha256 } },
|
||||
compiledHotPathHashes: Object.fromEntries(['server/dist/hot-a.js', 'server/dist/hot-b.js', 'server/dist/hot-c.js'].map(file => [file, hash(file)])) };
|
||||
options.evidence?.(evidence);
|
||||
await Promise.all([
|
||||
writeFile(path.join(directory, sidecar.manifestFile), manifestBytes),
|
||||
writeFile(path.join(directory, sidecar.inventorySummaryFile), summaryBytes),
|
||||
writeFile(path.join(directory, sidecar.qualifiedReceiptFile), receiptBytes),
|
||||
writeFile(path.join(directory, 'arm-inventory.json'), JSON.stringify(sidecar)),
|
||||
]);
|
||||
return { sidecarFile: path.join(directory, 'arm-inventory.json'), sidecar, evidence,
|
||||
owningPlaywright: { reportSha256 }, manifest, summary, receipt };
|
||||
}
|
||||
|
||||
async function withFixture(options, work) {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'h5-inventory-'));
|
||||
try { await work(await fixture(directory, options), directory); }
|
||||
finally { await rm(directory, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
test('full frozen inventory, qualified native build and owning report bind one arm', async () => {
|
||||
await withFixture({}, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.manifestFileSha256, value.sidecar.manifestFileSha256);
|
||||
assert.equal(proof.aggregateSha256, aggregate(descriptors()));
|
||||
assert.equal(proof.fileCount, descriptors().length);
|
||||
assert.equal(proof.qualifiedReceiptSha256, value.sidecar.qualifiedReceiptSha256);
|
||||
assert.equal(proof.playwrightReportSha256, reportSha256);
|
||||
assert.equal(proof.binarySha256, binarySha256);
|
||||
assert.deepEqual(proof.roots, roots);
|
||||
});
|
||||
});
|
||||
|
||||
test('exact compressed manifest bytes and build-receipt bytes cannot be swapped', async () => {
|
||||
for (const field of ['manifestFile', 'qualifiedReceiptFile']) await withFixture({}, async (value, directory) => {
|
||||
const file = path.join(directory, value.sidecar[field]);
|
||||
await writeFile(file, Buffer.concat([await readFile(file), Buffer.from(' ')]));
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /bytes changed/);
|
||||
});
|
||||
});
|
||||
|
||||
test('source, compiled build and owning report must match the arm', async () => {
|
||||
const cases = [
|
||||
[{ sidecar: row => { row.sourceRevision = '3'.repeat(40); } }, /source revision differs/],
|
||||
[{ sidecar: row => { row.compiledBuildRevision = '3'.repeat(40); } }, /build revision differs/],
|
||||
[{ sidecar: row => { row.playwrightReportSha256 = hash('another-report'); } }, /another owning Playwright/],
|
||||
[{ manifest: row => { row.revision = '3'.repeat(40); } }, /manifest source revision/],
|
||||
[{ receipt: row => { row.results[0].revision = '3'.repeat(40); } }, /receipt source revision/],
|
||||
[{ receipt: row => { row.results[0].compiledBuildRevision = '3'.repeat(40); } }, /receipt build revision/],
|
||||
[{ receipt: row => { row.results[0].sourceClean = false; } }, /source was not clean/],
|
||||
];
|
||||
for (const [options, message] of cases) await withFixture(options, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('file descriptors must be complete, bounded, distinct and match the Python aggregate', async () => {
|
||||
const cases = [
|
||||
[row => { row.fileCount--; }, /file count is incomplete/],
|
||||
[row => { row.aggregateSha256 = hash('wrong aggregate'); }, /aggregate does not match/],
|
||||
[row => { row.files[0].bytes = -1; }, /file descriptor/],
|
||||
[row => { row.files[0].path = '../outside'; }, /escapes registered roots/],
|
||||
[row => { row.files[1].path = row.files[0].path; }, /Duplicate/],
|
||||
[row => { row.files.reverse(); }, /order differs/],
|
||||
[row => { row.files.pop(); row.fileCount--; row.aggregateSha256 = aggregate(row.files); }, /omitted a registered root/],
|
||||
[row => { row.roots = ['server/dist']; }, /registered inventory roots/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ manifest: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('summary identifies exact gzip, count and aggregate for the selected variant', async () => {
|
||||
for (const [key, value] of [['manifestSha256', hash('wrong')], ['fileCount', 1], ['aggregateSha256', hash('wrong')]]) {
|
||||
await withFixture({ summary: row => { row.inventories.B2[key] = value; } }, async fixture => {
|
||||
await assert.rejects(validateArmInventory(fixture.sidecarFile, fixture), /Inventory summary/);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('serving entry, every hot path and native binary must match full evidence', async () => {
|
||||
const cases = [
|
||||
[row => { row.serverEntrySha256 = hash('wrong-entry'); }, /server entry differs/],
|
||||
[row => { row.compiledHotPathHashes['server/dist/hot-c.js'] = hash('wrong-hot-path'); }, /hot path differs/],
|
||||
[row => { row.remote.runner.sha256 = hash('wrong-native'); }, /native binary differs/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ evidence: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
await withFixture({ receipt: row => { row.nativeStageHashes.push({ ...row.nativeStageHashes[0] }); } }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /native stage exactly once/);
|
||||
});
|
||||
});
|
||||
|
||||
test('a future candidate revision is accepted only with matching complete provenance', async () => {
|
||||
await withFixture({ revision: 'f'.repeat(40) }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.revision, 'f'.repeat(40));
|
||||
assert.equal(proof.compiledBuildRevision, 'f'.repeat(40));
|
||||
});
|
||||
});
|
||||
|
||||
test('aggregate follows the existing Python ensure_ascii convention for non-ASCII paths', async () => {
|
||||
// Independently computed with the retained artifact-inventory-ce33.py
|
||||
// json.dumps(sort_keys=True, separators=(',', ':')) canonical convention.
|
||||
const expected = '6f7bf8cab07e2aaf8faaa95a8bdb565c588670e4e3e3580acdba1f93154fedde';
|
||||
await withFixture({ manifest: row => {
|
||||
row.files.at(-1).path = 'ui/dist/é😀\u007f.html'; row.aggregateSha256 = expected;
|
||||
} }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.aggregateSha256, expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { createRequire } from 'node:module';
|
||||
import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate,
|
||||
H5_LIMITS, ORIGINAL_BUDGETS } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
|
||||
const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`),
|
||||
{ matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } });
|
||||
|
||||
test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
assert.equal(require('@playwright/test/package.json').version, '1.62.1');
|
||||
await assert.rejects(expect.poll(async () => 0, { timeout: 15, intervals: [5] }).toBeGreaterThanOrEqual(1), error => {
|
||||
assert.equal(error.matcherResult?.name, 'toBeGreaterThanOrEqual');
|
||||
assert.equal(error.matcherResult.pass, false);
|
||||
assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate'));
|
||||
return true;
|
||||
});
|
||||
// This checks the real library's format; it does not execute a120s wait or
|
||||
// claim runtime qualification of H5's extended application path.
|
||||
});
|
||||
|
||||
test('H5 requires explicit mode and leaves default H4c selected', () => {
|
||||
assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true);
|
||||
assert.throws(() => h5Enabled('almost-h5'));
|
||||
});
|
||||
|
||||
test('first successful observation uses only the original wait with no collection delay', async () => {
|
||||
const calls = [];
|
||||
const value = await observeStreaming({ readCount: async () => 16, target: 16, now: () => 1000,
|
||||
poll: async (read, milliseconds, target) => { calls.push(milliseconds); assert.ok(await read() >= target); } });
|
||||
assert.deepEqual(calls, [120000]); assert.equal(value.original.status, 'passed');
|
||||
assert.equal(value.observation.complete, true); assert.equal(value.observation.completedAtMs, 1000);
|
||||
});
|
||||
|
||||
test('known120s miss remains failed while the same observation finishes before600s', async () => {
|
||||
let clock = 1000, count = 30; const calls = [], records = [];
|
||||
const value = await observeStreaming({ readCount: async () => count, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => {
|
||||
calls.push(milliseconds); await read();
|
||||
if (calls.length === 1) { clock += milliseconds; throw timeout(milliseconds); }
|
||||
count = 512; clock += 90000; await read();
|
||||
} });
|
||||
assert.deepEqual(calls, [120000, 480000]);
|
||||
assert.equal(value.original.status, 'failed'); assert.equal(value.original.lastObservedCount, 30);
|
||||
assert.equal(value.original.lastObservedAtMs, 1000); assert.equal(value.original.deadlineMs, 121000);
|
||||
assert.equal(value.observation.completedAtMs, 211000); assert.equal(records[0].observation.complete, false);
|
||||
assert.equal(value.original.reason, 'original_streaming_deadline');
|
||||
});
|
||||
|
||||
test('page read failures and unknown poll failures never become an expected SLA miss', async () => {
|
||||
const failure = new Error('page detached');
|
||||
await assert.rejects(observeStreaming({ readCount: async () => { throw failure; }, target: 1,
|
||||
poll: async read => { try { await read(); } catch {} throw timeout(120000); } }), error => error === failure);
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 0, target: 1,
|
||||
poll: async () => { throw new Error('unrelated assertion'); } }), /Only the original streaming matcher deadline/);
|
||||
});
|
||||
|
||||
test('observation ceiling leaves a censored failure, never a substituted latency', async () => {
|
||||
let clock = 1000, calls = 0; const records = [];
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 30, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => { await read(); calls++; clock += milliseconds; throw timeout(milliseconds); } }), /Timeout/);
|
||||
assert.equal(calls, 2); assert.equal(clock, 601000);
|
||||
assert.equal(records.at(-1).observation.complete, false); assert.equal(records.at(-1).observation.censored, true);
|
||||
assert.equal(records.at(-1).observation.completedAtMs, null);
|
||||
});
|
||||
|
||||
test('original terminal is not reached after120s miss; observational180s is distinct', () => {
|
||||
const terminal = { complete: true, startedAtMs: 211000, deadlineMs: 391000, assessedAtMs: 212000 };
|
||||
assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), {
|
||||
status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null });
|
||||
assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed');
|
||||
assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
});
|
||||
|
||||
test('cursor inherits only paired exact known misses; final candidate inherits none', () => {
|
||||
const base = { misses: ['burst512:streaming120'] };
|
||||
requireH5Candidate('cursor-nonregression', base, base);
|
||||
const slowInvocation = { misses: ['invocation:observed-total-duration900'] };
|
||||
requireH5Candidate('cursor-nonregression', slowInvocation, slowInvocation);
|
||||
requireH5Candidate('final-b2', base, { misses: [] });
|
||||
for (const candidate of [{ misses: ['paced:streaming120'] }, { misses: ['invocation:observed-total-duration900'] },
|
||||
{ misses: ['operator-stop:streaming120'] }]) assert.throws(() => requireH5Candidate('cursor-nonregression', base, candidate));
|
||||
assert.throws(() => requireH5Candidate('final-b2', base, base), /every original SLA/);
|
||||
assert.throws(() => requireH5Candidate('final-b2', slowInvocation, slowInvocation), /every original SLA/);
|
||||
});
|
||||
@@ -1,10 +1,12 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { H5_MODE, H5_PROTOCOL, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, originalTerminal } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
|
||||
test("four pairs must individually improve beyond noise", () => {
|
||||
@@ -161,6 +163,196 @@ function browserReport(revision, latency) {
|
||||
}) };
|
||||
}
|
||||
|
||||
function h5Report(revision, loadedLatency) {
|
||||
const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000);
|
||||
Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE,
|
||||
workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true,
|
||||
actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture",
|
||||
serverEntrySha256: "e".repeat(64), compiledHotPathHashes: {
|
||||
"server/dist/services/heartbeat-run-events.js": "f".repeat(64),
|
||||
"server/dist/services/native-runtime/paperclip-control-plane-port.js": "1".repeat(64),
|
||||
"server/dist/vendor/paperclip-runner/native-session-runtime.js": "2".repeat(64) } });
|
||||
for (const item of report.cases) {
|
||||
const latency = item.name === "burst512" ? loadedLatency : 1000;
|
||||
item.settledAtMs = item.epochMs + latency; item.drainThroughTerminalMs = latency;
|
||||
item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency };
|
||||
item.intendedToVisible = { p95Ms: latency };
|
||||
if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1;
|
||||
item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs }));
|
||||
const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs;
|
||||
const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null,
|
||||
startedAtMs: item.epochMs, deadlineMs: item.epochMs + 120000, assessedAtMs,
|
||||
target: item.stop ? 3 : item.events, lastObservedCount: missed ? 30 : item.visible.length,
|
||||
lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs,
|
||||
error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null },
|
||||
observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } };
|
||||
const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000,
|
||||
assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs,
|
||||
lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false };
|
||||
item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation),
|
||||
visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length,
|
||||
independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10,
|
||||
deltaCount: item.runs * (item.stop ? 3 : item.events), resultAcceptedCount: item.stop ? 0 : item.runs, clockStreamsClosed: true } };
|
||||
item.terminalStatuses = item.runIds.map(runId => ({ runId, runtimeMode: "native", status: item.stop ? "cancelled" : "succeeded" }));
|
||||
for (const row of item.durable) row.resultAcceptedEvents = item.stop ? 0 : 1;
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires complete owning evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; };
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" });
|
||||
const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map();
|
||||
async function inventory(report, playwrightFile, name) {
|
||||
const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision];
|
||||
if (!inventories.has(variant)) {
|
||||
const roots = ["server/dist", "ui/dist", "packages/db/src", "packages/shared/src"];
|
||||
const files = Object.entries({ ...report.compiledHotPathHashes, "server/dist/index.js": report.serverEntrySha256,
|
||||
"packages/db/src/client.ts": "3".repeat(64), "packages/shared/src/index.ts": "4".repeat(64), "ui/dist/index.html": "5".repeat(64) })
|
||||
.map(([path, sha256]) => ({ bytes: 10, path, sha256 })).sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
const aggregateSha256 = hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join("\n") + "\n");
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify({ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, roots, fileCount: files.length, files, aggregateSha256 })));
|
||||
const manifestFile = path.join(directory, `${variant}-artifact-inventory.json.gz`); await writeFile(manifestFile, manifestBytes);
|
||||
const manifestFileSha256 = hash(manifestBytes);
|
||||
const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: {
|
||||
manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } });
|
||||
const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40),
|
||||
binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }],
|
||||
artifactInventorySummary: inventorySummaryFile });
|
||||
inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile,
|
||||
inventorySummarySha256: hash(await readFile(inventorySummaryFile)), qualifiedReceiptFile,
|
||||
qualifiedReceiptSha256: hash(await readFile(qualifiedReceiptFile)) });
|
||||
}
|
||||
return save(`${name}-inventory.json`, { schema: "paperclip.native-ingestion.arm-inventory.v1", variant,
|
||||
sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision,
|
||||
playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) });
|
||||
}
|
||||
async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) {
|
||||
const entries = [];
|
||||
for (let index = 0; index < 5; index++) {
|
||||
const entry = {};
|
||||
for (const arm of ["baseline", "candidate"]) {
|
||||
const report = h5Report(arm === "baseline" || sameRevision ? baselineRevision : candidateRevision, arm === "baseline" ? 180000 : candidateLatency);
|
||||
const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0,
|
||||
duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] };
|
||||
if (index === 0 && arm === "candidate") mutate?.(report, owner);
|
||||
owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json",
|
||||
body: Buffer.from(JSON.stringify(report)).toString("base64") });
|
||||
entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report);
|
||||
entry[`${arm}PlaywrightFile`] = await save(`${prefix}-${index}-${arm}-playwright.json`, { suites: [{ tests: [owner] }] });
|
||||
entry[`${arm}InventoryFile`] = await inventory(report, entry[`${arm}PlaywrightFile`], `${prefix}-${index}-${arm}`);
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
return save(`${prefix}-pairs.json`, entries);
|
||||
}
|
||||
try {
|
||||
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
|
||||
proof.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream";
|
||||
proof.measurementSha256 = "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b";
|
||||
proof.harnessProvenance = { sourceRevision: "7d7e7a88d562a95a1d407b76474763b666c181f5", sourceStatus: "" };
|
||||
for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field];
|
||||
for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; }
|
||||
const originalProof = await save("original-h4c-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{
|
||||
status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{
|
||||
name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] });
|
||||
const originalInventory = await inventory(proof, originalProof, "original-h4c");
|
||||
const finalProofArgs = [`--original-h4c-proof=${originalProof}`, `--original-h4c-inventory=${originalInventory}`];
|
||||
const noise = path.join(directory, "aa.json");
|
||||
const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]);
|
||||
assert.equal(aa.status, 0, aa.stderr);
|
||||
const aaReport = JSON.parse(await readFile(noise, "utf8"));
|
||||
assert.equal(aaReport.h5Assessments.length, 5);
|
||||
assert.deepEqual(aaReport.h5Assessments[0].baseline.misses, ["burst512:streaming120"]);
|
||||
const cursor = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor", 180000, { candidateRevision: "B1" })}`, `--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor.status, 0, cursor.stderr);
|
||||
const cursor900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-matched900", 180000, { candidateRevision: "B1", duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor900.status, 0, cursor900.stderr);
|
||||
const added900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-added900", 180000, { candidateRevision: "B1", mutate: (_report, owner) => { owner.results[0].duration = 950000; } })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(added900.status, 0, "Cursor candidate cannot add an unmatched900 miss");
|
||||
const goodPairs = await manifest("good", 1000);
|
||||
const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(good.status, 0, good.stderr);
|
||||
const wrongPolicy = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", `--pairs=${goodPairs}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(wrongPolicy.status, 0); assert.match(wrongPolicy.stderr, /Cursor policy requires B0→B1/);
|
||||
const b1Noise = path.join(directory, "aa-b1.json");
|
||||
const b1AA = run(["--phase=aa", `--pairs=${await manifest("aa-b1", 181000, { sameRevision: true, baselineRevision: "B1" })}`, `--output=${b1Noise}`]);
|
||||
assert.equal(b1AA.status, 0, b1AA.stderr);
|
||||
const incremental = run(["--candidate-policy=final-b2", `--pairs=${await manifest("b1-b2", 1000, { baselineRevision: "B1" })}`,
|
||||
`--noise-report=${b1Noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(incremental.status, 0, incremental.stderr);
|
||||
const pendingFile = path.join(directory, "pending-h4c.json");
|
||||
const pending = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, `--output=${pendingFile}`, "--require-acceptance"]);
|
||||
assert.notEqual(pending.status, 0);
|
||||
const pendingReport = JSON.parse(await readFile(pendingFile, "utf8"));
|
||||
assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false);
|
||||
const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.equal(performanceOnly.status, 0, performanceOnly.stderr);
|
||||
for (const [name, change, message] of [
|
||||
["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/],
|
||||
["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/],
|
||||
["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /unchanged original H4c/],
|
||||
]) {
|
||||
const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0],
|
||||
value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString());
|
||||
change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64");
|
||||
const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`);
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
|
||||
`--original-h4c-proof=${file}`, `--original-h4c-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]);
|
||||
assert.notEqual(attempt.status, 0, name);
|
||||
const rejected = JSON.parse(await readFile(output, "utf8"));
|
||||
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
|
||||
assert.match(rejected.originalH4cConfirmation.reason, message);
|
||||
}
|
||||
const badOutput = path.join(directory, "still-misses.json");
|
||||
const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`,
|
||||
`--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]);
|
||||
assert.notEqual(bad.status, 0);
|
||||
const rejected = JSON.parse(await readFile(badOutput, "utf8"));
|
||||
assert.equal(rejected.acceptancePassed, false);
|
||||
assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false);
|
||||
for (const [name, mutate, message] of [
|
||||
["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/],
|
||||
["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/],
|
||||
["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/],
|
||||
["missing-delta", report => { report.cases[3].durable[99].individualDeltas--; }, /individualDeltas|result.events|512/],
|
||||
["wrong-workload", report => { report.workloadSha256 = "old"; }, /registration changed/],
|
||||
["pw-retry", (_report, owner) => { owner.results[0].retry = 1; }, /Retries/],
|
||||
["pw-error", (_report, owner) => { owner.results[0].errors.push({ message: "unexpected API failure" }); }, /deep-equal/],
|
||||
["wrong-observation-timeout", (_report, owner) => { owner.timeout = 900000; }, /1200000/],
|
||||
]) {
|
||||
const result = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutate })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(result.status, 0, name); assert.match(result.stderr, message, name);
|
||||
}
|
||||
const outer = run(["--candidate-policy=final-b2", `--pairs=${await manifest("outer900", 1000, { duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(outer.status, 0, "Observed total duration900 miss cannot qualify final B2");
|
||||
const incomplete = JSON.parse(await readFile(await manifest("missing-owner", 1000), "utf8"));
|
||||
delete incomplete[0].candidatePlaywrightFile;
|
||||
const missing = run(["--candidate-policy=final-b2", `--pairs=${await save("missing-owner-manifest.json", incomplete)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(missing.status, 0); assert.match(missing.stderr, /independent owning Playwright/);
|
||||
const global = JSON.parse(await readFile(await manifest("global-error", 1000), "utf8"));
|
||||
const globalOwner = JSON.parse(await readFile(global[0].candidatePlaywrightFile, "utf8"));
|
||||
globalOwner.errors = [{ message: "global teardown failed" }];
|
||||
await writeFile(global[0].candidatePlaywrightFile, JSON.stringify(globalOwner));
|
||||
const globalFailed = run(["--candidate-policy=final-b2", `--pairs=${await save("global-error-manifest.json", global)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(globalFailed.status, 0); assert.match(globalFailed.stderr, /global error/);
|
||||
} finally { await rm(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# H5 complete observation registration (prospective; no execution release)
|
||||
|
||||
H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5`
|
||||
(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`).
|
||||
Its observed 120-second streaming failures remain failures. Aborted or profiled
|
||||
attempts cannot be reused as complete timing arms. H5 repairs observation of the
|
||||
final outcome; it does not optimize the product or turn those failures into passes.
|
||||
|
||||
The fresh application baselines remain B0 `a36567f9fbf805ad72207d54f539516e3121741e`
|
||||
and B1 `d8cd3be0aeed4f9fb93ecc3cf725f1d46d5a0310`. Current B2
|
||||
`914129afb141d81c0d6d8c5f0ea11c20bd69a3da` has an observed original120s failure;
|
||||
it is not already qualified. A later candidate uses its exact new immutable
|
||||
revision, complete build receipt and inventory; no old binary is presumed equal.
|
||||
|
||||
Select H5 explicitly with
|
||||
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5-observe-original-sla`.
|
||||
The default code path retains the H4c 120/180/900-second behavior. H5 emits a new
|
||||
protocol version, hashes its added helper with the driver sources, and records a
|
||||
separate registration/workload SHA. The original comparator mode rejects H5.
|
||||
|
||||
Run `node scripts/describe-native-ingestion-h5.mjs` to inspect the registration.
|
||||
This is a dry run only: it starts no app, provider, database or browser and
|
||||
accepts no execution arguments. It grants no runtime release.
|
||||
|
||||
## Fixed work and observation boundaries
|
||||
|
||||
Keep the five streaming cases and original local Stop case exactly as H4c:
|
||||
paced 1×128 at25ms; burst 1×16; burst10×128; burst100×512; existing-history
|
||||
1000 foreground prelude deltas plus measured10×128; local Stop1×512 at100ms,
|
||||
clicked after at least three observed markers. Keep normal API/UI activation,
|
||||
waves of two, shared300s readiness, actual GET+clock hellos, verified TLS,
|
||||
closed READ ONLY REPEATABLE READ authority snapshot, source identities and
|
||||
payloads, original nonawaited source-clock writes, and the existing2RAF observer.
|
||||
No normalized-event polling channel, per-event HTTP, injected provider receipt,
|
||||
raw recovery-policy change, diagnostic instrumentation or replacement app entry
|
||||
is added. Fake-plugin streaming remains separate from real-cloud Stop proof.
|
||||
|
||||
| Boundary | Original requirement | H5 observation |
|
||||
| --- | --- | --- |
|
||||
| Foreground markers | Original Playwright120s wait | Continue to an absolute600s ceiling from that same wait start |
|
||||
| All terminal runs |180s after foreground completion (and Stop click) | Same180s after eventual foreground completion; hard failure at the ceiling |
|
||||
| Stop acknowledgement and leases | Existing30s polls and ordering | Unchanged; full click→acknowledgement metric retained |
|
||||
| Invocation | Original900s test budget | Actual1200s observation timeout, separately disclosed |
|
||||
|
||||
The first120s poll is the original matcher. Only that exact known matcher
|
||||
deadline can extend observation. Page/API errors, other assertions, terminal
|
||||
failures and cleanup failures never become an allowed SLA miss. There is no
|
||||
collection wait when markers are available. The600s ceiling is absolute, so a
|
||||
120s miss leaves at most480s additional observation, not another600s.
|
||||
|
||||
Record the original wait start/deadline, actual last count and its observation
|
||||
time, assessment time and error before continuing. Keep the original failure
|
||||
after completion. The original120s clock starts before the common source epoch;
|
||||
it is not silently rebased to epoch+120s. Derive a separate paint-deadline count
|
||||
from retained2RAF timestamps; do not mislabel it as the original poll's count.
|
||||
|
||||
If120s fails, original terminal processing was **not reached**. Later180s
|
||||
terminal observations are separately labeled and cannot make the original
|
||||
protocol pass. Keep epoch→all-terminal drainage unshortened. An additional
|
||||
independent completion upper bound follows existing screenshot, reload,
|
||||
persistence, clock and lease checks; it includes verification overhead and is
|
||||
not an exact last-commit time.
|
||||
|
||||
The owning public Playwright JSON result supplies duration, startTime, retry,
|
||||
status, errors and configured timeout. Its observed total duration is an
|
||||
independent conservative assessment of the original900s budget; hooks/teardown
|
||||
can be treated differently by the original timeout. It is not an exact original
|
||||
timer-expiration observation. A final candidate must have total duration<900s
|
||||
in every arm and later pass one unchanged original H4c full invocation on the
|
||||
same final SUT revision and full qualified artifact inventory.
|
||||
|
||||
## Complete evidence and eligibility
|
||||
|
||||
H5 `observationComplete` means all six cases reached finite correct outcomes,
|
||||
with every expected participant and foreground marker. It does not mean H4c
|
||||
passed. Every remote run must succeed in native mode, retain every normalized
|
||||
delta's complete payload/identity/order, have exactly one independently read
|
||||
`run.result.accepted`, and pass the existing reload/result/clock/lease checks.
|
||||
The100-run case must independently verify all51200 deltas. Stop keeps its
|
||||
original cancellation, durable acknowledgement, local journal, PID/group/provider
|
||||
absence and lease checks; it must not acquire an accepted success result.
|
||||
|
||||
An observation ceiling, failed/cancelled remote run, missing event/result,
|
||||
unexpected API error, incomplete source clock, replay conflict or cleanup error
|
||||
is ineligible. Preserve its raw partial evidence and censoring state. Never
|
||||
substitute the ceiling as latency, compute a512-event tail from a visible prefix,
|
||||
discard a slow arm, retry it into the sample, cancel background work to make a
|
||||
pair finish, or hide work remaining after the measured interval.
|
||||
|
||||
Each comparison-manifest entry contains `baselineFile`, `candidateFile`,
|
||||
`baselinePlaywrightFile`, `candidatePlaywrightFile`, `baselineInventoryFile`, and
|
||||
`candidateInventoryFile`. The owning Playwright result must contain exactly that
|
||||
evidence attachment, one passed observation attempt, retry0, no errors, and the
|
||||
actual1200s timeout. An inventory sidecar binds that report hash to the exact
|
||||
gzip inventory bytes, full descriptor aggregate, summary and qualified build
|
||||
receipt hashes, source/build revision, and reported serving/native fingerprints.
|
||||
These frozen artifacts are evidence of the qualified build; the external owner
|
||||
still checks the live checkout/build inventories before each arm. A sidecar does
|
||||
not itself attest that the live filesystem stayed unchanged.
|
||||
|
||||
| Role | Allowed original failures | Required result |
|
||||
| --- | --- | --- |
|
||||
| Fresh B0/B1 A/A arms | Known burst512 streaming120 miss; explicit conservative observed-duration900 miss | Complete finite correct observation; root-reviewed noise suitability |
|
||||
| B0→B1 cursor-only candidate | Only an exact registered known miss also present in the paired baseline: burst512 streaming120 or conservative observed-duration900 | No additional deadline failures; latency/control nonregression plus separate incremental SQL evidence; never claim SLA fixed |
|
||||
| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later unchanged H4c proof |
|
||||
|
||||
Only burst512's streaming miss is registered as a known workload SLA failure.
|
||||
New failures in other workloads require diagnosis. The existing Stop poll
|
||||
ordering is preserved; no invented absolute click→ack30s requirement replaces it.
|
||||
|
||||
## Stage order and acceptance
|
||||
|
||||
Prospectively freeze **B0 A/A → B1 A/A → B0→B1 nonregression → B1→B2 → B0→B2**.
|
||||
Each stage has exactly five fresh alternating pairs, separate disposable state
|
||||
per arm, matched payloads/load/resource/cache policy, no reused baseline arms,
|
||||
and no concurrent builds/profilers/benchmarks on the measurement VM. Root reviews
|
||||
and freezes both complete A/A envelopes before any further product fix. Local
|
||||
product implementation can then proceed while the unchanged B0→B1 stage runs;
|
||||
the candidate VM build waits until that stage is quiet.
|
||||
|
||||
Keep the existing per-metric maximum absolute paired A/A envelope, median gate,
|
||||
and at least4/5 individual noise-exceeding loaded wins. Each loaded p95
|
||||
emission-to-visible, p95 intended-to-visible and terminal-drainage metric must
|
||||
improve; other fixed latency/control metrics must not regress outside noise.
|
||||
Do not infer p99 from sparse observations. There is no new fixed percentage A/A
|
||||
cutoff: root must reject unsuitable noisy/nonstationary calibration explicitly.
|
||||
|
||||
Use `--observation-mode=h5-observe-original-sla` for the comparator. Candidate
|
||||
comparisons also require `--candidate-policy=cursor-nonregression --expect=nonregression`
|
||||
or `--candidate-policy=final-b2`. Final comparisons expose `performancePassed`
|
||||
separately; `acceptancePassed` remains false until `--original-h4c-proof` and
|
||||
`--original-h4c-inventory` establish the later fresh original functional pass.
|
||||
That proof is full, unprofiled, retry0, original900s, on the exact final SUT
|
||||
revision/qualified inventory, with a new disposable company and a start time
|
||||
after all candidate arms. Prior failed H4c attempts remain in the record.
|
||||
Use the explicit H5 `--require-performance` gate for the two final comparison
|
||||
stages while that later proof is pending; this requires every performance and
|
||||
candidate original-SLA gate, but grants no final acceptance. After the one later
|
||||
original H4c qualification, recompute both reports against their unchanged paired
|
||||
manifests with the proof and `--require-acceptance`. This re-evaluates retained
|
||||
evidence; it neither reuses an arm in a different comparison nor reruns failures.
|
||||
|
||||
Before any runtime release, review the exact H5 protocol/comparator/helper
|
||||
hashes, registration dry run, outcome/eligibility tests, types and test listing.
|
||||
Then qualify the changed observation path before fresh A/A. Each later stage
|
||||
requires a separate root release pinning exact driver, build/inventory and
|
||||
reviewed noise-report bytes; verify the noise hash before the first arm and
|
||||
again before comparison. Product or workload changes require new prospective
|
||||
provenance and affected requalification; do not loosen gates after results.
|
||||
|
||||
Separate successful full-window diagnostics may later assess Node/PG CPU,
|
||||
queue/backlog, delay and memory with identical capture hooks. Aborted profiles
|
||||
remain diagnostic-only and cannot establish complete CPU/event costs. No CPU
|
||||
requirement or profiling is added to this timing comparator. Local results do
|
||||
not establish deployed improvement.
|
||||
@@ -0,0 +1,176 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const H5_MODE = 'h5-observe-original-sla';
|
||||
export const H5_PROTOCOL = 'v5-complete-observation-original-sla';
|
||||
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'a36567f9fbf805ad72207d54f539516e3121741e',
|
||||
B1: 'd8cd3be0aeed4f9fb93ecc3cf725f1d46d5a0310' });
|
||||
export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000,
|
||||
streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 });
|
||||
export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });
|
||||
export const H5_WORKLOADS = Object.freeze([
|
||||
{ name: 'paced', runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: 'burst16', runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'burst128', runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'burst512', runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'existing-history', runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false },
|
||||
{ name: 'operator-stop', runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
|
||||
]);
|
||||
export const H5_REGISTRATION = Object.freeze({ mode: H5_MODE, protocolVersion: H5_PROTOCOL,
|
||||
baselineRevisions: H5_BASELINE_REVISIONS,
|
||||
originalBudgets: ORIGINAL_BUDGETS, observationLimits: H5_LIMITS, workloads: H5_WORKLOADS,
|
||||
pairs: 5, order: 'baseline-first on even pair index; candidate-first on odd pair index',
|
||||
stages: ['B0 A/A', 'B1 A/A', 'B0→B1 nonregression', 'B1→B2 improvement', 'B0→B2 improvement'],
|
||||
knownBaselineMisses: ['burst512:streaming120', 'invocation:observed-total-duration900'],
|
||||
baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of H4c',
|
||||
cursorCandidateEligibility: 'Nonregression; only the paired baseline exact registered known misses may be inherited',
|
||||
finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later unchanged H4c full functional PASS required',
|
||||
noise: 'Fresh B0 and B1 A/A frozen before any further product implementation; no old abort/profile samples',
|
||||
performance: 'Loaded p95 emission/intended-to-visible and terminal drainage improve beyond frozen noise in median and at least4/5 pairs; all other metrics nonregress',
|
||||
});
|
||||
export const H5_WORKLOAD_SHA256 = createHash('sha256').update(JSON.stringify(H5_REGISTRATION)).digest('hex');
|
||||
|
||||
export function h5Enabled(value) {
|
||||
assert.ok(value === undefined || value === H5_MODE, 'Explicit supported H5 observation mode required');
|
||||
return value === H5_MODE;
|
||||
}
|
||||
|
||||
// Keep the original Playwright poll as the first wait. Only its known matcher
|
||||
// deadline may extend observation; callback errors and other failures propagate.
|
||||
export async function observeStreaming({ readCount, poll, target, now = Date.now, onOutcome = _value => {} }) {
|
||||
const startedAtMs = now();
|
||||
let lastObservedCount = null, lastObservedAtMs = null, readError;
|
||||
const read = async () => {
|
||||
try {
|
||||
const count = await readCount();
|
||||
assert.ok(Number.isSafeInteger(count) && count >= 0, 'Invalid visible marker count');
|
||||
lastObservedCount = count; lastObservedAtMs = now(); return count;
|
||||
} catch (error) { readError = error; throw error; }
|
||||
};
|
||||
let deadlineError;
|
||||
try { await poll(read, ORIGINAL_BUDGETS.streamingMs, target); }
|
||||
catch (error) {
|
||||
if (readError) throw readError;
|
||||
assert.ok(error?.matcherResult?.name === 'toBeGreaterThanOrEqual' && error.matcherResult.pass === false
|
||||
&& error.message.includes(`Timeout ${ORIGINAL_BUDGETS.streamingMs}ms exceeded while waiting on the predicate`),
|
||||
'Only the original streaming matcher deadline can extend observation');
|
||||
assert.ok(lastObservedCount !== null && lastObservedCount < target, 'Deadline needs an incomplete actual observation');
|
||||
deadlineError = error;
|
||||
}
|
||||
if (readError) throw readError;
|
||||
const original = { status: deadlineError ? 'failed' : 'passed', reason: deadlineError ? 'original_streaming_deadline' : null,
|
||||
startedAtMs, deadlineMs: startedAtMs + ORIGINAL_BUDGETS.streamingMs, assessedAtMs: now(),
|
||||
target, lastObservedCount, lastObservedAtMs, error: deadlineError?.message ?? null };
|
||||
const observation = { ceilingMs: H5_LIMITS.streamingMs, deadlineMs: startedAtMs + H5_LIMITS.streamingMs,
|
||||
complete: false, censored: false, completedAtMs: null };
|
||||
const result = { original, observation };
|
||||
onOutcome(result);
|
||||
if (deadlineError) {
|
||||
try {
|
||||
const remaining = observation.deadlineMs - now();
|
||||
assert.ok(remaining > 0, 'H5 streaming observation ceiling exceeded');
|
||||
await poll(read, remaining, target);
|
||||
} catch (error) { observation.censored = true; onOutcome(result); throw readError ?? error; }
|
||||
}
|
||||
if (readError) throw readError;
|
||||
observation.complete = true; observation.completedAtMs = now(); onOutcome(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function originalTerminal(streaming, terminal) {
|
||||
return streaming.original.status === 'passed'
|
||||
? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null : 'original_terminal_deadline',
|
||||
startedAtMs: terminal.startedAtMs, deadlineMs: terminal.deadlineMs, assessedAtMs: terminal.assessedAtMs }
|
||||
: { status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null };
|
||||
}
|
||||
|
||||
export function assessH5Arm(report, owner) {
|
||||
assert.equal(report.protocolVersion, H5_PROTOCOL);
|
||||
assert.equal(report.observationMode, H5_MODE);
|
||||
assert.equal(report.observationComplete, true, 'H5 requires complete finite observation');
|
||||
assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false);
|
||||
assert.equal(report.diagnosticOnly, false); assert.equal(report.performanceAcceptanceEligible, true);
|
||||
assert.equal(report.cleanup?.complete, true); assert.deepEqual(report.cleanup.errors, []);
|
||||
assert.equal(report.workloadSha256, H5_WORKLOAD_SHA256, 'H5 registration changed');
|
||||
assert.deepEqual(report.observationLimits, H5_LIMITS);
|
||||
assert.deepEqual(report.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(report.actualPlaywrightTimeoutMs, H5_LIMITS.fullInvocationMs);
|
||||
assert.deepEqual(report.cases.map(item => item.name), H5_WORKLOADS.map(item => item.name), 'Complete fixed six-case observation required');
|
||||
assert.equal(owner.status, 'passed', 'Owning Playwright attempt must pass observation/correctness');
|
||||
assert.equal(owner.retry, 0, 'Retries cannot replace an H5 arm');
|
||||
assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, H5_LIMITS.fullInvocationMs);
|
||||
assert.ok(Number.isFinite(owner.durationMs) && owner.durationMs > 0 && owner.durationMs < H5_LIMITS.fullInvocationMs);
|
||||
assert.ok(typeof owner.startedAt === 'string' && Number.isFinite(Date.parse(owner.startedAt)));
|
||||
const misses = [];
|
||||
for (const item of report.cases) {
|
||||
const value = item.h5;
|
||||
assert.ok(value, 'Missing H5 case outcomes');
|
||||
assert.equal(value.streaming.observation.complete, true);
|
||||
assert.equal(value.streaming.observation.censored, false);
|
||||
const streaming = value.streaming.original;
|
||||
for (const field of ['startedAtMs', 'deadlineMs', 'assessedAtMs', 'lastObservedAtMs'])
|
||||
assert.ok(Number.isSafeInteger(streaming[field]) && streaming[field] > 0, 'Missing raw original-deadline timing');
|
||||
assert.ok(['passed', 'failed'].includes(streaming.status));
|
||||
assert.equal(streaming.deadlineMs, streaming.startedAtMs + ORIGINAL_BUDGETS.streamingMs);
|
||||
assert.equal(streaming.target, item.stop ? 3 : item.events);
|
||||
assert.ok(Number.isSafeInteger(streaming.lastObservedCount) && streaming.lastObservedCount >= 0);
|
||||
assert.ok(streaming.lastObservedAtMs >= streaming.startedAtMs && streaming.lastObservedAtMs <= streaming.assessedAtMs);
|
||||
if (streaming.status === 'failed') {
|
||||
assert.equal(streaming.reason, 'original_streaming_deadline');
|
||||
assert.ok(streaming.lastObservedCount < streaming.target);
|
||||
assert.ok(streaming.assessedAtMs >= streaming.deadlineMs);
|
||||
assert.equal(item.stop, false, 'Control readiness cannot inherit a streaming SLA miss');
|
||||
assert.ok(streaming.error.includes('Timeout 120000ms exceeded while waiting on the predicate'));
|
||||
assert.equal(value.terminalOriginal.status, 'not-reached');
|
||||
assert.equal(value.terminalOriginal.reason, 'original_streaming_deadline');
|
||||
misses.push(`${item.name}:streaming120`);
|
||||
} else {
|
||||
assert.ok(streaming.lastObservedCount >= streaming.target);
|
||||
assert.equal(streaming.reason, null);
|
||||
assert.equal(streaming.error, null);
|
||||
assert.equal(value.terminalOriginal.status, 'passed');
|
||||
}
|
||||
assert.equal(value.terminalObservation.complete, true, 'H5 terminal observation must drain');
|
||||
assert.equal(value.terminalObservation.censored, false);
|
||||
assert.equal(value.terminalObservation.deadlineMs, value.terminalObservation.startedAtMs + H5_LIMITS.terminalDrainageMs);
|
||||
assert.equal(value.terminalObservation.assessedAtMs, item.settledAtMs);
|
||||
assert.equal(value.terminalObservation.observedRuns, item.runs); assert.equal(value.terminalObservation.terminalRuns, item.runs);
|
||||
assert.ok(value.terminalObservation.lastObservedAtMs >= value.terminalObservation.startedAtMs
|
||||
&& value.terminalObservation.lastObservedAtMs <= value.terminalObservation.assessedAtMs);
|
||||
assert.deepEqual(value.terminalObservation.lastStatuses.map(row => row.runId).sort(), [...item.runIds].sort());
|
||||
assert.ok(value.terminalObservation.lastStatuses.every(row => row.status === (item.stop ? 'cancelled' : 'succeeded')));
|
||||
assert.equal(value.streaming.observation.ceilingMs, H5_LIMITS.streamingMs);
|
||||
assert.equal(value.streaming.observation.deadlineMs, streaming.startedAtMs + H5_LIMITS.streamingMs);
|
||||
assert.ok(value.streaming.observation.completedAtMs >= streaming.assessedAtMs
|
||||
&& value.streaming.observation.completedAtMs <= value.streaming.observation.deadlineMs);
|
||||
assert.ok(value.terminalObservation.startedAtMs >= value.streaming.observation.completedAtMs);
|
||||
assert.ok(item.settledAtMs <= value.terminalObservation.deadlineMs);
|
||||
assert.deepEqual(value.terminalOriginal, originalTerminal(value.streaming, value.terminalObservation));
|
||||
assert.equal(value.visibleCountByOriginalPaintDeadline, item.visible.filter(marker => marker.visibleMs <= streaming.deadlineMs).length);
|
||||
assert.deepEqual(item.terminalStatuses.map(row => row.runId).sort(), [...item.runIds].sort());
|
||||
assert.ok(item.terminalStatuses.every(row => row.runtimeMode === 'native' && row.status === (item.stop ? 'cancelled' : 'succeeded')),
|
||||
'Failed terminal runs are not successful drainage');
|
||||
assert.ok(item.durable.every(row => row.resultAcceptedEvents === (item.stop ? 0 : 1)), 'Every completed run needs its independently persisted result');
|
||||
assert.ok(value.independentDrain.verifiedAtMs >= item.settledAtMs);
|
||||
assert.equal(value.independentDrain.elapsedFromEpochMs, value.independentDrain.verifiedAtMs - item.epochMs);
|
||||
assert.equal(value.independentDrain.deltaCount, item.durable.reduce((sum, row) => sum + row.individualDeltas, 0));
|
||||
assert.equal(value.independentDrain.resultAcceptedCount, item.stop ? 0 : item.runs);
|
||||
assert.equal(value.independentDrain.clockStreamsClosed, true);
|
||||
}
|
||||
if (owner.durationMs >= ORIGINAL_BUDGETS.fullInvocationMs) misses.push('invocation:observed-total-duration900');
|
||||
return { originalSlaPassed: misses.length === 0, misses,
|
||||
original900budget: { status: owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs ? 'passed' : 'failed',
|
||||
assessedAgainst: 'observed owning Playwright total duration', durationMs: owner.durationMs,
|
||||
limitMs: ORIGINAL_BUDGETS.fullInvocationMs,
|
||||
scope: 'Independent conservative gate; total duration can include hooks/teardown that the original timeout treats differently; not an exact original timer-expiration fact' } };
|
||||
}
|
||||
|
||||
export function requireH5Candidate(policy, baseline, candidate) {
|
||||
if (policy === 'final-b2') assert.deepEqual(candidate.misses, [], 'Final B2 must pass every original SLA in every arm');
|
||||
else if (policy === 'cursor-nonregression') {
|
||||
const allowed = new Set(H5_REGISTRATION.knownBaselineMisses);
|
||||
assert.ok(candidate.misses.every(miss => allowed.has(miss) && baseline.misses.includes(miss)),
|
||||
'Cursor-only candidate may inherit only the paired baseline exact registered known misses');
|
||||
} else throw new Error('Explicit final-b2 or cursor-nonregression candidate policy required');
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import { hostname } from "node:os";
|
||||
import path from "node:path";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { json } from "./agent-chat.shared";
|
||||
import { h5Enabled, H5_MODE, H5_PROTOCOL, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS,
|
||||
observeStreaming, originalTerminal } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
type SeenMarker = { markerCode: string; index: number; visibleMs: number };
|
||||
type Emission = { nonce: string; markerCode: string; index: number; intendedMs: number; emittedMs: number; receivedAtMs: number };
|
||||
@@ -23,6 +25,7 @@ type SetupActivation = { index: number; issueId: string; token: string; waveInde
|
||||
type SetupWave = { index: number; indices: number[]; startedAtMs: number; settledAtMs?: number;
|
||||
statuses?: Array<"fulfilled" | "rejected"> };
|
||||
const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1";
|
||||
const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE);
|
||||
const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "sandbox-remote-with-local-stop";
|
||||
const providerReadinessMs = 300_000;
|
||||
const setupActivationConcurrency = 2;
|
||||
@@ -150,6 +153,7 @@ const processAbsent = (pid: number) => {
|
||||
};
|
||||
|
||||
test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => {
|
||||
const bodyStartedAtMs = Date.now();
|
||||
const pending = new Map<string, Map<string, ServerResponse>>();
|
||||
const clocks = new Map<string, ClockCase>();
|
||||
let markerCounter = 0;
|
||||
@@ -271,10 +275,13 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video),
|
||||
screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" };
|
||||
const report: Record<string, unknown> = {
|
||||
protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport,
|
||||
protocolVersion: h5 ? H5_PROTOCOL : "v4-remote-streaming-original-local-stop-clock-stream", transport,
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS,
|
||||
observationComplete: false, bodyStartedAtMs, actualPlaywrightTimeoutMs: info.timeout,
|
||||
originalInvocationBudgetAssessment: "Requires independent owning Playwright observed total duration; conservative original900 assessment, not exact timeout expiration" } : {}),
|
||||
transportPolicy: { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" },
|
||||
resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000,
|
||||
resourceBudgets: h5 ? ORIGINAL_BUDGETS : { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000,
|
||||
terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 },
|
||||
admissionProtocol: "two-at-a-time-provider-and-clock-readiness-before-common-epoch",
|
||||
setupActivationConcurrency,
|
||||
@@ -307,7 +314,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex");
|
||||
const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs",
|
||||
"fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs",
|
||||
"fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs"];
|
||||
"fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs",
|
||||
...(h5 ? ["fixtures/native-ingestion-h5.mjs"] : [])];
|
||||
const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) })));
|
||||
report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex");
|
||||
const harnessRoot = path.resolve(import.meta.dirname, "../..");
|
||||
@@ -529,7 +537,12 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
active.participants = participants;
|
||||
for (const response of pending.get(key)!.values()) response.writeHead(200, { "Content-Type": "application/json" }).end(JSON.stringify({ epochMs }));
|
||||
const observed = () => page.evaluate(code => (window as unknown as { nativeIngestionVisible: SeenMarker[] }).nativeIngestionVisible.filter(item => item.markerCode === code), markerCode);
|
||||
await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events);
|
||||
let streamingObservation: Awaited<ReturnType<typeof observeStreaming>> | undefined;
|
||||
if (h5) streamingObservation = await observeStreaming({ readCount: async () => (await observed()).length,
|
||||
target: stop ? 3 : workload.events,
|
||||
poll: (read: () => Promise<number>, timeout: number, target: number) => expect.poll(read, { timeout }).toBeGreaterThanOrEqual(target),
|
||||
onOutcome: (value: unknown) => { active.streamingObservation = value; } });
|
||||
else await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events);
|
||||
let stopRequestedAt: number | undefined;
|
||||
if (stop) {
|
||||
const button = page.getByRole("button", { name: "Stop", exact: true });
|
||||
@@ -539,11 +552,25 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
}
|
||||
active.phase = "terminal-drainage";
|
||||
let runs: any[] = [];
|
||||
await expect.poll(async () => {
|
||||
const terminalStartedAtMs = h5 ? Date.now() : undefined;
|
||||
const terminalObservation = h5 ? { startedAtMs: terminalStartedAtMs!, deadlineMs: terminalStartedAtMs! + H5_LIMITS.terminalDrainageMs,
|
||||
assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0,
|
||||
lastStatuses: [] as Array<{ runId: string; status: string }>, complete: false, censored: false } : undefined;
|
||||
if (h5) active.terminalObservation = terminalObservation;
|
||||
try { await expect.poll(async () => {
|
||||
runs = (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId));
|
||||
if (terminalObservation) Object.assign(terminalObservation, { lastObservedAtMs: Date.now(), observedRuns: runs.length,
|
||||
terminalRuns: runs.filter(run => ["succeeded", "cancelled", "failed"].includes(run.status)).length,
|
||||
lastStatuses: runs.map(run => ({ runId: run.id, status: run.status })) });
|
||||
return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status));
|
||||
}, { timeout: 180_000 }).toBe(true);
|
||||
}, { timeout: 180_000 }).toBe(true); }
|
||||
catch (error) {
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = Date.now(); terminalObservation.censored = true;
|
||||
active.terminalOriginal = originalTerminal(streamingObservation!, terminalObservation); }
|
||||
throw error;
|
||||
}
|
||||
const settledAt = Date.now(), seenMarkers = await observed();
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = settledAt; terminalObservation.complete = true; }
|
||||
active.settledAt = settledAt; active.seenMarkers = seenMarkers;
|
||||
const details = [];
|
||||
for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`)));
|
||||
@@ -618,6 +645,11 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs };
|
||||
const clockEvidence: Array<Record<string, unknown>> = [];
|
||||
const result = { ...workload, stop, transport: caseTransport, environmentId: caseEnvironment.id,
|
||||
...(h5 ? { h5: { streaming: streamingObservation, terminalObservation,
|
||||
terminalOriginal: originalTerminal(streamingObservation!, terminalObservation!),
|
||||
visibleCountByOriginalPaintDeadline: seenMarkers.filter(marker => marker.visibleMs <= streamingObservation!.original.deadlineMs).length,
|
||||
deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : {}),
|
||||
...(h5 ? { terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status })) } : {}),
|
||||
setupActivationConcurrency, setupReadinessStartedAtMs, readinessDeadlineMs, allReadyObservedAtMs, activations, activationWaves,
|
||||
nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt,
|
||||
stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps,
|
||||
@@ -648,16 +680,19 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const code = state.streams.get(token)!.markerCode;
|
||||
expect(token).toBeTruthy();
|
||||
const deltas: Array<{ index: number; identity: string }> = [];
|
||||
let afterSeq = 0;
|
||||
let afterSeq = 0, resultAcceptedEvents = 0;
|
||||
while (true) {
|
||||
const rows = await json(await request.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
|
||||
if (!rows.length) break;
|
||||
for (const row of rows) {
|
||||
const event = row.payload?.prpEvent;
|
||||
if (h5 && (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted")) resultAcceptedEvents++;
|
||||
if (event?.eventType !== "item.delta") continue;
|
||||
for (const match of String(event.payload?.text ?? "").matchAll(new RegExp(`\\[${code}([a-z0-9]{1,3})\\]`, "g"))) {
|
||||
const index = Number.parseInt(match[1], 36);
|
||||
expect(event.payload.text, "Every durable normalized delta retains its complete compact payload").toBe(`[${code}${index.toString(36)}] `);
|
||||
if (h5) { expect(event.runId).toBe(run.id); expect(event.sourceInstanceId).toBeTruthy();
|
||||
expect(event.sourceEventId).toBeTruthy(); expect(Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > 0).toBe(true); }
|
||||
deltas.push({ index, identity: `${event.sourceInstanceId}:${event.sourceEventId}:${event.sourceSeq}` });
|
||||
}
|
||||
}
|
||||
@@ -674,11 +709,21 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
expect(new Set(deltas.map(item => item.identity)).size).toBe(deltas.length);
|
||||
expect(deltas.map(item => item.index)).toEqual(Array.from({ length: stop ? deltas.length : workload.events }, (_, index) => index));
|
||||
if (stop) for (const item of visible) expect(deltas.some(delta => delta.index === item.index)).toBe(true);
|
||||
durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, matchedSourceClocks: deltas.length, ordered: true });
|
||||
if (h5) expect(resultAcceptedEvents).toBe(stop ? 0 : 1);
|
||||
durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length,
|
||||
matchedSourceClocks: deltas.length, ordered: true, ...(h5 ? { resultAcceptedEvents } : {}) });
|
||||
}
|
||||
await expect.poll(() => [...state.streams.values()].every(stream => stream.ended || stream.aborted), { timeout: 30_000 }).toBe(true);
|
||||
expect(state.errors).toEqual([]);
|
||||
expect([...state.streams.values()].every(stream => !stream.error)).toBe(true);
|
||||
if (h5) {
|
||||
const verifiedAtMs = Date.now();
|
||||
Object.assign(result.h5!, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs,
|
||||
deltaCount: durable.reduce((count, row) => count + row.individualDeltas, 0),
|
||||
resultAcceptedCount: durable.reduce((count, row) => count + (row.resultAcceptedEvents ?? 0), 0),
|
||||
clockStreamsClosed: true,
|
||||
scope: "Independent completion upper bound after existing reload/API/clock/lease checks; includes verification and screenshot overhead, not exact last-commit time" } });
|
||||
}
|
||||
clockEvidence.push(...[...state.streams.values()].map(stream => ({ nonce: stream.nonce, markerCode: stream.markerCode,
|
||||
launcher: stream.launcher, providerPid: stream.providerPid,
|
||||
hello: stream.hello, ended: stream.ended, aborted: stream.aborted, error: stream.error, partialBytes: stream.partialBytes,
|
||||
@@ -691,6 +736,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
for (const workload of cases) await perform(workload);
|
||||
await perform({ name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0 }, true);
|
||||
report.complete = true;
|
||||
if (h5) report.observationComplete = true;
|
||||
} catch (error) {
|
||||
report.primaryFailure = { name: error instanceof Error ? error.name : "UnknownError",
|
||||
message: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined };
|
||||
@@ -728,7 +774,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
await attemptCleanup("restore-experimental-settings", async () => json(await request.patch("/api/instance/settings/experimental", {
|
||||
data: { enableNativeRunner: original.enableNativeRunner, enableWorktreeRunExecution: original.enableWorktreeRunExecution } })));
|
||||
report.cleanup = { errors: cleanupErrors, complete: cleanupErrors.length === 0 };
|
||||
if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; }
|
||||
if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false;
|
||||
if (h5) report.observationComplete = false; }
|
||||
if (h5) report.bodyFinishedAtMs = Date.now();
|
||||
await info.attach("native-ingestion-evidence", { body: JSON.stringify(report, null, 2), contentType: "application/json" });
|
||||
if (!report.primaryFailure) expect(cleanupErrors, "API cleanup must succeed before accepting an invocation").toEqual([]);
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { defineConfig, type ReporterDescription } from "@playwright/test";
|
||||
import base from "./playwright.config";
|
||||
import { h5Enabled, H5_LIMITS } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
const server = base.webServer as Exclude<typeof base.webServer, unknown[]>;
|
||||
const evidence = process.env.PAPERCLIP_INGESTION_EVIDENCE_DIR;
|
||||
const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE);
|
||||
const browserRecording = Boolean(process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR) || process.env.PAPERCLIP_INGESTION_BROWSER_CAPTURE === "1";
|
||||
process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "off";
|
||||
const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../.."));
|
||||
@@ -33,7 +35,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-"
|
||||
export default defineConfig({
|
||||
...base,
|
||||
testMatch: "native-event-ingestion.journey.ts",
|
||||
timeout: 900_000,
|
||||
timeout: h5 ? H5_LIMITS.fullInvocationMs : 900_000,
|
||||
...(evidence ? { outputDir: path.join(evidence, "test-results"),
|
||||
reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}),
|
||||
use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 },
|
||||
|
||||
Reference in new issue
Block a user