From 81c9ffdf0358cc9173a04ec32ef60d29cbb8cdab Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 21:44:55 -0600 Subject: [PATCH] test: observe full ingestion drainage with original deadline outcomes --- scripts/compare-native-ingestion-browser.ts | 155 +++++++++++++- scripts/describe-native-ingestion-h5.mjs | 7 + scripts/native-ingestion-h5-inventory.mjs | 137 +++++++++++++ .../native-ingestion-h5-inventory.test.mjs | 154 ++++++++++++++ scripts/native-ingestion-h5.test.mjs | 88 ++++++++ ...ative-ingestion-performance-gates.test.mjs | 192 ++++++++++++++++++ tests/e2e/fixtures/native-ingestion-h5.md | 155 ++++++++++++++ tests/e2e/fixtures/native-ingestion-h5.mjs | 176 ++++++++++++++++ tests/e2e/native-event-ingestion.journey.ts | 66 +++++- .../e2e/playwright-native-ingestion.config.ts | 4 +- 10 files changed, 1120 insertions(+), 14 deletions(-) create mode 100644 scripts/describe-native-ingestion-h5.mjs create mode 100644 scripts/native-ingestion-h5-inventory.mjs create mode 100644 scripts/native-ingestion-h5-inventory.test.mjs create mode 100644 scripts/native-ingestion-h5.test.mjs create mode 100644 tests/e2e/fixtures/native-ingestion-h5.md create mode 100644 tests/e2e/fixtures/native-ingestion-h5.mjs diff --git a/scripts/compare-native-ingestion-browser.ts b/scripts/compare-native-ingestion-browser.ts index 354e7b0ef37..4d34d7d127d 100644 --- a/scripts/compare-native-ingestion-browser.ts +++ b/scripts/compare-native-ingestion-browser.ts @@ -3,6 +3,8 @@ import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; +import { H5_MODE, H5_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs"; +import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs"; // Each manifest entry points at native-ingestion-evidence JSON attachments // from two isolated compiled-app runs. Run the variants in alternating order; @@ -11,12 +13,87 @@ const arg = (name: string) => process.argv.find(value => value.startsWith(`--${n const manifestPath = arg("pairs"); assert.ok(manifestPath, "Pass --pairs="); const phase = arg("phase") ?? "compare", expectation = arg("expect") ?? "improvement"; +const h5 = arg("observation-mode") === H5_MODE; +assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode"); +const candidatePolicy = arg("candidate-policy"); assert.ok(["aa", "compare"].includes(phase)); assert.ok(["improvement", "nonregression"].includes(expectation)); -const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string }>; +const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string; + baselinePlaywrightFile?: string; candidatePlaywrightFile?: string; baselineInventoryFile?: string; candidateInventoryFile?: string }>; assert.ok(manifest.length >= 5, "At least five actual paired application runs are required"); +if (h5) assert.equal(manifest.length, 5, "H5 requires exactly five prospectively registered pairs"); const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8")); -const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) }))); +async function owningResult(reportPath: string, evidence?: any) { + const bytes = await readFile(reportPath); + const report = JSON.parse(bytes.toString("utf8")), tests: any[] = []; + assert.deepEqual(report.errors ?? [], [], "Owning Playwright report has a global error"); + const visit = (value: any) => { + if (Array.isArray(value)) for (const child of value) visit(child); + else if (value && typeof value === "object") { + if (Array.isArray(value.results) && Object.hasOwn(value, "expectedStatus")) tests.push(value); + for (const [key, child] of Object.entries(value)) if (key !== "results") visit(child); + } + }; + visit(report); assert.equal(tests.length, 1, "One complete owning test is required; no partial selection"); + const test = tests[0]; assert.equal(test.results.length, 1, "No retry or omitted owning attempt"); + assert.equal(test.expectedStatus, "passed"); + const result = test.results[0], attachments = result.attachments.filter((item: any) => item.name === "native-ingestion-evidence"); + assert.ok(result.error === undefined || result.error === null, "Owning Playwright attempt has an error"); + assert.equal(attachments.length, 1, "Owning result must carry this exact evidence attachment"); + const attachment = attachments[0]; + const attached = attachment.body ? JSON.parse(Buffer.from(attachment.body, "base64").toString("utf8")) + : JSON.parse(await readFile(path.resolve(path.dirname(reportPath), attachment.path), "utf8")); + if (evidence) assert.deepEqual(attached, evidence, "Playwright result belongs to a different evidence arm"); + return { evidence: attached, owner: { reportFile: reportPath, reportSha256: createHash("sha256").update(bytes).digest("hex"), + status: result.status, retry: result.retry, errors: result.errors, startedAt: result.startTime, + durationMs: result.duration, timeoutMs: test.timeout } }; +} +async function loadArm(file: string, playwrightFile?: string, inventoryFile?: string) { + const evidence = await load(file); + if (!h5) return evidence; + assert.ok(playwrightFile, "H5 requires each arm's independent owning Playwright JSON report"); + evidence.owningPlaywright = (await owningResult(path.resolve(path.dirname(manifestPath!), playwrightFile), evidence)).owner; + assert.ok(inventoryFile, "H5 requires each arm's exact full artifact inventory sidecar"); + evidence.armInventory = await validateArmInventory(path.resolve(path.dirname(manifestPath!), inventoryFile), + { evidence, owningPlaywright: evidence.owningPlaywright }); + return evidence; +} +const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await loadArm(entry.baselineFile, entry.baselinePlaywrightFile, entry.baselineInventoryFile), + candidate: await loadArm(entry.candidateFile, entry.candidatePlaywrightFile, entry.candidateInventoryFile) }))); +const h5Assessments: any[] = []; +if (h5) { + if (phase === "compare") { + assert.ok(["final-b2", "cursor-nonregression"].includes(candidatePolicy ?? ""), "H5 requires an explicit candidate SLA policy"); + assert.equal(expectation, candidatePolicy === "cursor-nonregression" ? "nonregression" : "improvement"); + } + for (const pair of pairs) { + assert.ok(Object.values(H5_BASELINE_REVISIONS).includes(pair.baseline.sourceRevision), "H5 baseline must use the prospectively frozen B0 or B1 revision"); + const baselineVariant = pair.baseline.sourceRevision === H5_BASELINE_REVISIONS.B0 ? "B0" : "B1"; + assert.equal(pair.baseline.armInventory.variant, baselineVariant, "Baseline inventory role differs from its frozen revision"); + if (phase === "aa") assert.equal(pair.candidate.armInventory.variant, baselineVariant, "A/A must use the same baseline inventory role"); + if (phase === "compare" && candidatePolicy === "cursor-nonregression") { + assert.equal(pair.baseline.sourceRevision, H5_BASELINE_REVISIONS.B0, "Cursor policy requires B0→B1"); + assert.equal(pair.candidate.sourceRevision, H5_BASELINE_REVISIONS.B1, "Cursor policy requires B0→B1"); + assert.equal(pair.candidate.armInventory.variant, "B1", "Cursor policy cannot stand in for final B2 acceptance"); + } else if (phase === "compare") { + assert.equal(pair.candidate.armInventory.variant, "B2", "Final policy requires the qualified B2 inventory role"); + assert.ok(!Object.values(H5_BASELINE_REVISIONS).includes(pair.candidate.sourceRevision), "Final B2 must differ from both frozen baselines"); + } + const baseline = assessH5Arm(pair.baseline, pair.baseline.owningPlaywright); + const candidate = assessH5Arm(pair.candidate, pair.candidate.owningPlaywright); + // The only prospectively known baseline workload SLA miss is loaded streaming. + const allowed = new Set(["burst512:streaming120", "invocation:observed-total-duration900"]); + for (const miss of baseline.misses) assert.ok(allowed.has(miss), "Unknown baseline deadline failure needs diagnosis"); + if (phase === "aa") for (const miss of candidate.misses) assert.ok(allowed.has(miss), "Unknown A/A deadline failure needs diagnosis"); + let candidateSlaPolicyError; + if (phase === "compare") { + try { requireH5Candidate(candidatePolicy, baseline, candidate); } + catch (error) { candidateSlaPolicyError = error instanceof Error ? error.message : String(error); } + } + h5Assessments.push({ baseline, candidate, candidateSlaPolicyError, candidateSlaPolicyPassed: !candidateSlaPolicyError, + owningPlaywright: { baseline: pair.baseline.owningPlaywright, candidate: pair.candidate.owningPlaywright } }); + } +} const workloads = [ { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false }, { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false }, @@ -32,7 +109,8 @@ const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/ const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value); for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) { const report = pair[arm]; - assert.equal(report.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream", "Fresh browser acceptance requires the qualified v4 mixed clock protocol"); + assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : "v4-remote-streaming-original-local-stop-clock-stream", + h5 ? "Fresh browser acceptance requires its qualified H5 observation protocol" : "Fresh browser acceptance requires the qualified v4 mixed clock protocol"); assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport"); assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local", realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, "Missing explicit transport and real cloud Stop qualification policy"); @@ -97,6 +175,8 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm"); assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples"); assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples"); + if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) + assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples"); assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs"); if (phase === "aa") assert.equal(report.sourceRevision, first.baseline.sourceRevision, "A/A must use one source revision"); for (const name of names) { @@ -300,6 +380,56 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`); } } +let originalH4cConfirmation: Record | undefined; +if (h5 && phase === "compare" && candidatePolicy === "final-b2") { + originalH4cConfirmation = { required: true, passed: false, reason: "Later unchanged original H4c full functional proof is pending" }; + if (arg("original-h4c-proof")) { + try { + assert.ok(arg("original-h4c-inventory"), "Original H4c proof requires its full artifact inventory sidecar"); + const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4c-proof")!)); + assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []); + assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs); + assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs); + assert.equal(proof.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream"); + assert.equal(proof.observationMode, undefined, "Confirmation must use unchanged original H4c defaults"); + assert.equal(proof.harnessProvenance?.sourceRevision, "7d7e7a88d562a95a1d407b76474763b666c181f5"); + assert.equal(proof.harnessProvenance.sourceStatus, ""); + assert.equal(proof.measurementSha256, "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b"); + assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS); + assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false); + assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled"); + assert.equal(proof.cleanup?.complete, true); assert.deepEqual(proof.cleanup.errors, []); + assert.deepEqual(proof.browserRecording, first.candidate.browserRecording); + assert.equal(proof.sourceRevision, first.candidate.sourceRevision); assert.equal(proof.compiledBuildRevision, first.candidate.compiledBuildRevision); + assert.equal(proof.serverEntrySha256, first.candidate.serverEntrySha256); + assert.deepEqual(proof.compiledHotPathHashes, first.candidate.compiledHotPathHashes); + assert.equal(proof.transport, first.candidate.transport); assert.deepEqual(proof.transportPolicy, first.candidate.transportPolicy); + assert.deepEqual(proof.cases.map((item: any) => item.name), names, "Original confirmation must include all six workloads"); + for (const fixed of workloads) { + const item = proof.cases.find((item: any) => item.name === fixed.name); + for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(item[field], fixed[field]); + assert.equal(item.persistedAfterReload, true); assert.equal(item.durable.length, fixed.runs); + assert.equal(item.matchedVisibleClocks, item.visible.length); + assert.ok(item.visible.length >= (fixed.stop ? 3 : fixed.events)); + assert.equal(item.runIds.length, fixed.runs); assert.equal(new Set(item.runIds).size, fixed.runs); + for (const row of item.durable) { + assert.equal(row.ordered, true); assert.equal(row.uniqueSourceIdentities, row.individualDeltas); + assert.equal(row.matchedSourceClocks, row.individualDeltas); + if (!fixed.stop) assert.equal(row.individualDeltas, fixed.events); + } + } + const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs)); + assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms"); + assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database"); + const inventory = await validateArmInventory(path.resolve(arg("original-h4c-inventory")!), { evidence: proof, owningPlaywright: owner }); + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const) + assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory"); + originalH4cConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision }; + } catch (error) { + originalH4cConfirmation.reason = error instanceof Error ? error.message : String(error); + } + } +} const metric = (item: any, key: string): number => key.split(".").reduce((value, part) => value[part], item); const measurements = names.flatMap(name => { const observed = first.baseline.cases.find((item: any) => item.name === name); @@ -334,11 +464,21 @@ const result: Record = { comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"), manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements, primaryLoadedWorkload: "burst512", complete: true, + ...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments, + artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4cConfirmation, + originalProtocolAcceptance: "Baseline SLA misses are retained; H5 complete observation is not H4c PASS", + finalUnchangedH4cFunctionalPassRequired: candidatePolicy === "final-b2" } : {}), }; if (phase === "compare") { assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report"); const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8")); assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true); + if (h5) { + assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256); + assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required"); + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) + assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A"); + } assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol"); assert.equal(noise.transport, result.transport, "Noise report uses a different transport"); assert.deepEqual(noise.transportPolicy, result.transportPolicy, "Transport policy changed after A/A calibration"); @@ -362,10 +502,17 @@ if (phase === "compare") { noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins, passed: improvementRequired ? improvement : noRegression }; }); - result.acceptancePassed = result.gates.every((entry: any) => entry.passed); + result.performancePassed = result.gates.every((entry: any) => entry.passed) + && (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed)); + result.acceptancePassed = result.performancePassed && (!originalH4cConfirmation || originalH4cConfirmation.passed); } if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2)); +if (process.argv.includes("--require-performance")) { + assert.equal(h5, true, "H5 performance-only gating is explicit; original H4c remains unchanged"); + assert.equal(phase, "compare"); + assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose"); +} if (process.argv.includes("--require-acceptance")) { assert.equal(phase, "compare", "A/A calibrates noise; it is not performance acceptance"); assert.equal(result.acceptancePassed, true, "Browser performance acceptance failed; retain results and diagnose"); diff --git a/scripts/describe-native-ingestion-h5.mjs b/scripts/describe-native-ingestion-h5.mjs new file mode 100644 index 00000000000..8d4bf55bde2 --- /dev/null +++ b/scripts/describe-native-ingestion-h5.mjs @@ -0,0 +1,7 @@ +import { H5_REGISTRATION, H5_WORKLOAD_SHA256 } from '../tests/e2e/fixtures/native-ingestion-h5.mjs'; +if (process.argv.length > 2) throw new Error('Dry run only: this command accepts no execution arguments'); +console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadSha256: H5_WORKLOAD_SHA256, + registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes', + 'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation', + 'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass', + 'Later unchanged original H4c full functional pass on the final SUT'] }, null, 2)); diff --git a/scripts/native-ingestion-h5-inventory.mjs b/scripts/native-ingestion-h5-inventory.mjs new file mode 100644 index 00000000000..e97bfc76d03 --- /dev/null +++ b/scripts/native-ingestion-h5-inventory.mjs @@ -0,0 +1,137 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; +import { gunzipSync } from 'node:zlib'; + +const SCHEMA = 'paperclip.native-ingestion.arm-inventory.v1'; +const ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src']; +const sha256 = value => createHash('sha256').update(value).digest('hex'); +const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +const revision = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value); +const object = value => value && typeof value === 'object' && !Array.isArray(value); + +async function bounded(file, maximumBytes) { + const metadata = await stat(file); + assert.ok(metadata.isFile() && metadata.size <= maximumBytes, 'Inventory evidence file is absent, nonregular or oversized'); + const bytes = await readFile(file); + assert.ok(bytes.length <= maximumBytes, 'Inventory evidence file grew beyond its bound'); + return bytes; +} + +function relativeArtifact(value) { + return typeof value === 'string' && value.length > 0 && !value.includes('\\') + && !value.includes('\0') && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value + && !value.split('/').includes('..') && ROOTS.some(root => value.startsWith(`${root}/`)); +} + +// The existing Python generator sorts Unicode code points and uses json.dumps +// with sort_keys=True, separators=(',', ':') and its default ensure_ascii=True. +function pythonOrder(left, right) { + const a = [...left], b = [...right]; + for (let index = 0; index < Math.min(a.length, b.length); index++) { + const difference = a[index].codePointAt(0) - b[index].codePointAt(0); + if (difference) return difference; + } + return a.length - b.length; +} +const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 }) + .replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`); + +/** + * Validates frozen inventory evidence and reported serving fingerprints only. + * The owner supplies this sidecar per arm; this helper never reads the SUT tree + * and does not attest that a live filesystem remained unchanged during a run. + */ +export async function validateArmInventory(sidecarFile, { evidence, owningPlaywright }) { + assert.equal(typeof sidecarFile, 'string', 'Missing arm inventory sidecar'); + assert.ok(object(evidence) && object(owningPlaywright), 'Missing serving evidence or owning Playwright report'); + const absoluteSidecar = path.resolve(sidecarFile), base = path.dirname(absoluteSidecar); + const sidecarBytes = await bounded(absoluteSidecar, 256 * 1024); + const sidecar = JSON.parse(sidecarBytes.toString('utf8')); + assert.equal(sidecar.schema, SCHEMA, 'Unsupported arm inventory sidecar schema'); + assert.ok(['B0', 'B1', 'B2'].includes(sidecar.variant), 'Unknown inventory variant'); + assert.ok(revision(sidecar.sourceRevision) && revision(sidecar.compiledBuildRevision), 'Invalid inventory source revision'); + assert.equal(sidecar.sourceRevision, evidence.sourceRevision, 'Inventory source revision differs from arm'); + assert.equal(sidecar.compiledBuildRevision, evidence.compiledBuildRevision, 'Inventory build revision differs from arm'); + assert.equal(sidecar.compiledBuildRevision, sidecar.sourceRevision, 'Inventory was built from a different source revision'); + assert.ok(digest(sidecar.playwrightReportSha256) && digest(owningPlaywright.reportSha256), 'Missing owning Playwright report hash'); + assert.equal(sidecar.playwrightReportSha256, owningPlaywright.reportSha256, 'Inventory belongs to another owning Playwright report'); + const resolve = name => { + assert.ok(typeof sidecar[name] === 'string' && sidecar[name].length > 0, `Missing inventory ${name}`); + return path.resolve(base, sidecar[name]); + }; + const readHashed = async (name, hashName, maximumBytes) => { + assert.ok(digest(sidecar[hashName]), `Invalid inventory ${hashName}`); + const file = resolve(name), bytes = await bounded(file, maximumBytes); + assert.equal(sha256(bytes), sidecar[hashName], `Inventory ${name} bytes changed`); + return { file, bytes }; + }; + const manifestSource = await readHashed('manifestFile', 'manifestFileSha256', 64 * 1024 * 1024); + const manifest = JSON.parse(gunzipSync(manifestSource.bytes, { maxOutputLength: 128 * 1024 * 1024 }).toString('utf8')); + assert.deepEqual(Object.keys(manifest).sort(), ['aggregateSha256', 'compiledBuildRevision', 'fileCount', 'files', 'revision', 'roots', 'variant']); + assert.equal(manifest.variant, sidecar.variant, 'Inventory manifest variant differs from sidecar'); + assert.equal(manifest.revision, sidecar.sourceRevision, 'Inventory manifest source revision differs from arm'); + assert.equal(manifest.compiledBuildRevision, sidecar.compiledBuildRevision, 'Inventory manifest build revision differs from arm'); + assert.deepEqual(manifest.roots, ROOTS, 'Full registered inventory roots are required'); + assert.ok(Number.isSafeInteger(manifest.fileCount) && manifest.fileCount > 0 && manifest.fileCount <= 500000, 'Invalid inventory file count'); + assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount, 'Inventory file count is incomplete'); + assert.ok(digest(manifest.aggregateSha256) && digest(sidecar.aggregateSha256), 'Invalid inventory aggregate hash'); + const files = new Map(); + for (const row of manifest.files) { + assert.ok(object(row), 'Invalid inventory file row'); + assert.deepEqual(Object.keys(row).sort(), ['bytes', 'path', 'sha256']); + assert.ok(relativeArtifact(row.path), 'Inventory file escapes registered roots'); + assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0 && digest(row.sha256), 'Invalid inventory file descriptor'); + assert.equal(files.has(row.path), false, 'Duplicate inventory file path'); + files.set(row.path, row); + } + const sorted = [...manifest.files].sort((left, right) => pythonOrder(left.path, right.path)); + assert.deepEqual(manifest.files.map(row => row.path), sorted.map(row => row.path), 'Inventory file order differs from generator'); + for (const root of ROOTS) assert.ok(manifest.files.some(row => row.path.startsWith(`${root}/`)), 'Inventory omitted a registered root'); + const aggregate = sha256(Buffer.from(`${sorted.map(pythonRow).join('\n')}\n`, 'utf8')); + assert.equal(aggregate, manifest.aggregateSha256, 'Inventory aggregate does not match exact file descriptors'); + assert.equal(aggregate, sidecar.aggregateSha256, 'Inventory aggregate differs from sidecar'); + assert.ok(digest(evidence.serverEntrySha256), 'Missing serving server entry hash'); + assert.equal(files.get('server/dist/index.js')?.sha256, evidence.serverEntrySha256, 'Serving server entry differs from full inventory'); + assert.ok(object(evidence.compiledHotPathHashes) && Object.keys(evidence.compiledHotPathHashes).length > 0, 'Missing serving hot-path fingerprints'); + for (const [file, hash] of Object.entries(evidence.compiledHotPathHashes)) { + assert.ok(relativeArtifact(file) && digest(hash), 'Invalid serving hot-path fingerprint'); + assert.equal(files.get(file)?.sha256, hash, 'Serving hot path differs from full inventory'); + } + const summarySource = await readHashed('inventorySummaryFile', 'inventorySummarySha256', 1024 * 1024); + const summary = JSON.parse(summarySource.bytes.toString('utf8')); + const summaryRow = summary.inventories?.[sidecar.variant]; + assert.ok(object(summaryRow), 'Inventory summary omitted this variant'); + assert.ok(typeof summaryRow.manifest === 'string' && path.basename(summaryRow.manifest) === summaryRow.manifest + && summaryRow.manifest.endsWith('.json.gz'), 'Inventory summary manifest reference is invalid'); + assert.equal(summaryRow.manifestSha256, sidecar.manifestFileSha256, 'Inventory summary points to different manifest bytes'); + assert.equal(summaryRow.aggregateSha256, aggregate, 'Inventory summary aggregate differs'); + assert.equal(summaryRow.fileCount, manifest.fileCount, 'Inventory summary file count differs'); + const receiptSource = await readHashed('qualifiedReceiptFile', 'qualifiedReceiptSha256', 4 * 1024 * 1024); + const receipt = JSON.parse(receiptSource.bytes.toString('utf8')); + assert.ok(Array.isArray(receipt.results), 'Qualified build receipt has no results'); + const qualified = receipt.results.filter(row => row.variant === sidecar.variant); + assert.equal(qualified.length, 1, 'Qualified build receipt must identify this variant exactly once'); + assert.equal(qualified[0].revision, sidecar.sourceRevision, 'Qualified build receipt source revision differs'); + assert.equal(qualified[0].compiledBuildRevision, sidecar.compiledBuildRevision, 'Qualified build receipt build revision differs'); + assert.equal(qualified[0].sourceClean, true, 'Qualified build source was not clean'); + assert.ok(typeof receipt.artifactInventorySummary === 'string' && receipt.artifactInventorySummary.length > 0, + 'Qualified build receipt omitted its inventory summary reference'); + assert.ok(revision(receipt.rustSourceTree), 'Qualified build receipt omitted Rust input identity'); + assert.ok(Array.isArray(receipt.nativeStageHashes), 'Qualified build receipt omitted native stage hashes'); + const native = receipt.nativeStageHashes.filter(row => row.variant === sidecar.variant); + assert.equal(native.length, 1, 'Qualified build receipt must identify this native stage exactly once'); + assert.ok(digest(native[0].binarySha256) && digest(receipt.binarySha256), 'Qualified native binary hash is invalid'); + assert.equal(native[0].binarySha256, receipt.binarySha256, 'Qualified native binary hashes disagree'); + assert.equal(native[0].binarySha256, evidence.remote?.runner?.sha256, 'Serving native binary differs from qualified build'); + return { schema: SCHEMA, sidecarFile: absoluteSidecar, sidecarFileSha256: sha256(sidecarBytes), variant: sidecar.variant, + revision: manifest.revision, compiledBuildRevision: manifest.compiledBuildRevision, + manifestFile: manifestSource.file, manifestFileSha256: sidecar.manifestFileSha256, + aggregateSha256: aggregate, fileCount: manifest.fileCount, roots: manifest.roots, + inventorySummaryFile: summarySource.file, inventorySummarySha256: sidecar.inventorySummarySha256, + qualifiedReceiptFile: receiptSource.file, qualifiedReceiptSha256: sidecar.qualifiedReceiptSha256, + binarySha256: native[0].binarySha256, rustSourceTree: receipt.rustSourceTree, + playwrightReportSha256: owningPlaywright.reportSha256, + scope: 'Exact frozen manifest/summary/build-receipt bytes for all registered roots, with reported server/hot-path/native fingerprints checked; no live SUT filesystem attestation' }; +} diff --git a/scripts/native-ingestion-h5-inventory.test.mjs b/scripts/native-ingestion-h5-inventory.test.mjs new file mode 100644 index 00000000000..25323d07894 --- /dev/null +++ b/scripts/native-ingestion-h5-inventory.test.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { gzipSync } from 'node:zlib'; +import { validateArmInventory } from './native-ingestion-h5-inventory.mjs'; + +const hash = value => createHash('sha256').update(value).digest('hex'); +const sourceRevision = '1'.repeat(40), reportSha256 = hash('owning-playwright'), binarySha256 = hash('runner'); +const roots = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src']; +const descriptors = () => [ + 'packages/db/src/client.ts', 'packages/shared/src/index.ts', 'server/dist/hot-a.js', + 'server/dist/hot-b.js', 'server/dist/hot-c.js', 'server/dist/index.js', 'ui/dist/index.html', +].map(file => ({ bytes: 10, path: file, sha256: hash(file) })); +const aggregate = files => hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join('\n') + '\n'); + +async function fixture(directory, options = {}) { + const revision = options.revision ?? sourceRevision; + const files = descriptors(), manifest = { variant: 'B2', revision, compiledBuildRevision: revision, + roots, fileCount: files.length, aggregateSha256: aggregate(files), files }; + options.manifest?.(manifest); + const manifestBytes = gzipSync(Buffer.from(JSON.stringify(manifest))); + const summary = { capturedAt: '2026-11-03T00:00:00.000Z', inventories: { B2: { + manifest: 'B2-artifact-inventory.json.gz', manifestSha256: hash(manifestBytes), fileCount: manifest.fileCount, + aggregateSha256: manifest.aggregateSha256 } } }; + options.summary?.(summary); + const receipt = { completedAt: '2026-11-03T00:00:01.000Z', + results: [{ variant: 'B2', revision, compiledBuildRevision: revision, sourceClean: true }], + rustSourceTree: '2'.repeat(40), binarySha256, nativeStageHashes: [{ variant: 'B2', binarySha256 }], + artifactInventorySummary: '/owned/original/artifact-inventory-summary.json' }; + options.receipt?.(receipt); + const summaryBytes = Buffer.from(JSON.stringify(summary)), receiptBytes = Buffer.from(JSON.stringify(receipt)); + const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant: 'B2', sourceRevision: revision, + compiledBuildRevision: revision, playwrightReportSha256: reportSha256, + manifestFile: 'B2-artifact-inventory.json.gz', manifestFileSha256: hash(manifestBytes), + aggregateSha256: manifest.aggregateSha256, + qualifiedReceiptFile: 'qualified.json', qualifiedReceiptSha256: hash(receiptBytes), + inventorySummaryFile: 'artifact-inventory-summary.json', inventorySummarySha256: hash(summaryBytes) }; + options.sidecar?.(sidecar); + const evidence = { sourceRevision: revision, compiledBuildRevision: revision, + serverEntrySha256: hash('server/dist/index.js'), remote: { runner: { sha256: binarySha256 } }, + compiledHotPathHashes: Object.fromEntries(['server/dist/hot-a.js', 'server/dist/hot-b.js', 'server/dist/hot-c.js'].map(file => [file, hash(file)])) }; + options.evidence?.(evidence); + await Promise.all([ + writeFile(path.join(directory, sidecar.manifestFile), manifestBytes), + writeFile(path.join(directory, sidecar.inventorySummaryFile), summaryBytes), + writeFile(path.join(directory, sidecar.qualifiedReceiptFile), receiptBytes), + writeFile(path.join(directory, 'arm-inventory.json'), JSON.stringify(sidecar)), + ]); + return { sidecarFile: path.join(directory, 'arm-inventory.json'), sidecar, evidence, + owningPlaywright: { reportSha256 }, manifest, summary, receipt }; +} + +async function withFixture(options, work) { + const directory = await mkdtemp(path.join(os.tmpdir(), 'h5-inventory-')); + try { await work(await fixture(directory, options), directory); } + finally { await rm(directory, { recursive: true, force: true }); } +} + +test('full frozen inventory, qualified native build and owning report bind one arm', async () => { + await withFixture({}, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.manifestFileSha256, value.sidecar.manifestFileSha256); + assert.equal(proof.aggregateSha256, aggregate(descriptors())); + assert.equal(proof.fileCount, descriptors().length); + assert.equal(proof.qualifiedReceiptSha256, value.sidecar.qualifiedReceiptSha256); + assert.equal(proof.playwrightReportSha256, reportSha256); + assert.equal(proof.binarySha256, binarySha256); + assert.deepEqual(proof.roots, roots); + }); +}); + +test('exact compressed manifest bytes and build-receipt bytes cannot be swapped', async () => { + for (const field of ['manifestFile', 'qualifiedReceiptFile']) await withFixture({}, async (value, directory) => { + const file = path.join(directory, value.sidecar[field]); + await writeFile(file, Buffer.concat([await readFile(file), Buffer.from(' ')])); + await assert.rejects(validateArmInventory(value.sidecarFile, value), /bytes changed/); + }); +}); + +test('source, compiled build and owning report must match the arm', async () => { + const cases = [ + [{ sidecar: row => { row.sourceRevision = '3'.repeat(40); } }, /source revision differs/], + [{ sidecar: row => { row.compiledBuildRevision = '3'.repeat(40); } }, /build revision differs/], + [{ sidecar: row => { row.playwrightReportSha256 = hash('another-report'); } }, /another owning Playwright/], + [{ manifest: row => { row.revision = '3'.repeat(40); } }, /manifest source revision/], + [{ receipt: row => { row.results[0].revision = '3'.repeat(40); } }, /receipt source revision/], + [{ receipt: row => { row.results[0].compiledBuildRevision = '3'.repeat(40); } }, /receipt build revision/], + [{ receipt: row => { row.results[0].sourceClean = false; } }, /source was not clean/], + ]; + for (const [options, message] of cases) await withFixture(options, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); +}); + +test('file descriptors must be complete, bounded, distinct and match the Python aggregate', async () => { + const cases = [ + [row => { row.fileCount--; }, /file count is incomplete/], + [row => { row.aggregateSha256 = hash('wrong aggregate'); }, /aggregate does not match/], + [row => { row.files[0].bytes = -1; }, /file descriptor/], + [row => { row.files[0].path = '../outside'; }, /escapes registered roots/], + [row => { row.files[1].path = row.files[0].path; }, /Duplicate/], + [row => { row.files.reverse(); }, /order differs/], + [row => { row.files.pop(); row.fileCount--; row.aggregateSha256 = aggregate(row.files); }, /omitted a registered root/], + [row => { row.roots = ['server/dist']; }, /registered inventory roots/], + ]; + for (const [change, message] of cases) await withFixture({ manifest: change }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); +}); + +test('summary identifies exact gzip, count and aggregate for the selected variant', async () => { + for (const [key, value] of [['manifestSha256', hash('wrong')], ['fileCount', 1], ['aggregateSha256', hash('wrong')]]) { + await withFixture({ summary: row => { row.inventories.B2[key] = value; } }, async fixture => { + await assert.rejects(validateArmInventory(fixture.sidecarFile, fixture), /Inventory summary/); + }); + } +}); + +test('serving entry, every hot path and native binary must match full evidence', async () => { + const cases = [ + [row => { row.serverEntrySha256 = hash('wrong-entry'); }, /server entry differs/], + [row => { row.compiledHotPathHashes['server/dist/hot-c.js'] = hash('wrong-hot-path'); }, /hot path differs/], + [row => { row.remote.runner.sha256 = hash('wrong-native'); }, /native binary differs/], + ]; + for (const [change, message] of cases) await withFixture({ evidence: change }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); + await withFixture({ receipt: row => { row.nativeStageHashes.push({ ...row.nativeStageHashes[0] }); } }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), /native stage exactly once/); + }); +}); + +test('a future candidate revision is accepted only with matching complete provenance', async () => { + await withFixture({ revision: 'f'.repeat(40) }, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.revision, 'f'.repeat(40)); + assert.equal(proof.compiledBuildRevision, 'f'.repeat(40)); + }); +}); + +test('aggregate follows the existing Python ensure_ascii convention for non-ASCII paths', async () => { + // Independently computed with the retained artifact-inventory-ce33.py + // json.dumps(sort_keys=True, separators=(',', ':')) canonical convention. + const expected = '6f7bf8cab07e2aaf8faaa95a8bdb565c588670e4e3e3580acdba1f93154fedde'; + await withFixture({ manifest: row => { + row.files.at(-1).path = 'ui/dist/é😀\u007f.html'; row.aggregateSha256 = expected; + } }, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.aggregateSha256, expected); + }); +}); diff --git a/scripts/native-ingestion-h5.test.mjs b/scripts/native-ingestion-h5.test.mjs new file mode 100644 index 00000000000..beda14a1280 --- /dev/null +++ b/scripts/native-ingestion-h5.test.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createRequire } from 'node:module'; +import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate, + H5_LIMITS, ORIGINAL_BUDGETS } from '../tests/e2e/fixtures/native-ingestion-h5.mjs'; + +const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`), + { matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } }); + +test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => { + const require = createRequire(import.meta.url), { expect } = require('@playwright/test'); + assert.equal(require('@playwright/test/package.json').version, '1.62.1'); + await assert.rejects(expect.poll(async () => 0, { timeout: 15, intervals: [5] }).toBeGreaterThanOrEqual(1), error => { + assert.equal(error.matcherResult?.name, 'toBeGreaterThanOrEqual'); + assert.equal(error.matcherResult.pass, false); + assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate')); + return true; + }); + // This checks the real library's format; it does not execute a120s wait or + // claim runtime qualification of H5's extended application path. +}); + +test('H5 requires explicit mode and leaves default H4c selected', () => { + assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true); + assert.throws(() => h5Enabled('almost-h5')); +}); + +test('first successful observation uses only the original wait with no collection delay', async () => { + const calls = []; + const value = await observeStreaming({ readCount: async () => 16, target: 16, now: () => 1000, + poll: async (read, milliseconds, target) => { calls.push(milliseconds); assert.ok(await read() >= target); } }); + assert.deepEqual(calls, [120000]); assert.equal(value.original.status, 'passed'); + assert.equal(value.observation.complete, true); assert.equal(value.observation.completedAtMs, 1000); +}); + +test('known120s miss remains failed while the same observation finishes before600s', async () => { + let clock = 1000, count = 30; const calls = [], records = []; + const value = await observeStreaming({ readCount: async () => count, target: 512, now: () => clock, + onOutcome: value => records.push(structuredClone(value)), + poll: async (read, milliseconds) => { + calls.push(milliseconds); await read(); + if (calls.length === 1) { clock += milliseconds; throw timeout(milliseconds); } + count = 512; clock += 90000; await read(); + } }); + assert.deepEqual(calls, [120000, 480000]); + assert.equal(value.original.status, 'failed'); assert.equal(value.original.lastObservedCount, 30); + assert.equal(value.original.lastObservedAtMs, 1000); assert.equal(value.original.deadlineMs, 121000); + assert.equal(value.observation.completedAtMs, 211000); assert.equal(records[0].observation.complete, false); + assert.equal(value.original.reason, 'original_streaming_deadline'); +}); + +test('page read failures and unknown poll failures never become an expected SLA miss', async () => { + const failure = new Error('page detached'); + await assert.rejects(observeStreaming({ readCount: async () => { throw failure; }, target: 1, + poll: async read => { try { await read(); } catch {} throw timeout(120000); } }), error => error === failure); + await assert.rejects(observeStreaming({ readCount: async () => 0, target: 1, + poll: async () => { throw new Error('unrelated assertion'); } }), /Only the original streaming matcher deadline/); +}); + +test('observation ceiling leaves a censored failure, never a substituted latency', async () => { + let clock = 1000, calls = 0; const records = []; + await assert.rejects(observeStreaming({ readCount: async () => 30, target: 512, now: () => clock, + onOutcome: value => records.push(structuredClone(value)), + poll: async (read, milliseconds) => { await read(); calls++; clock += milliseconds; throw timeout(milliseconds); } }), /Timeout/); + assert.equal(calls, 2); assert.equal(clock, 601000); + assert.equal(records.at(-1).observation.complete, false); assert.equal(records.at(-1).observation.censored, true); + assert.equal(records.at(-1).observation.completedAtMs, null); +}); + +test('original terminal is not reached after120s miss; observational180s is distinct', () => { + const terminal = { complete: true, startedAtMs: 211000, deadlineMs: 391000, assessedAtMs: 212000 }; + assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), { + status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null }); + assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed'); + assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs); +}); + +test('cursor inherits only paired exact known misses; final candidate inherits none', () => { + const base = { misses: ['burst512:streaming120'] }; + requireH5Candidate('cursor-nonregression', base, base); + const slowInvocation = { misses: ['invocation:observed-total-duration900'] }; + requireH5Candidate('cursor-nonregression', slowInvocation, slowInvocation); + requireH5Candidate('final-b2', base, { misses: [] }); + for (const candidate of [{ misses: ['paced:streaming120'] }, { misses: ['invocation:observed-total-duration900'] }, + { misses: ['operator-stop:streaming120'] }]) assert.throws(() => requireH5Candidate('cursor-nonregression', base, candidate)); + assert.throws(() => requireH5Candidate('final-b2', base, base), /every original SLA/); + assert.throws(() => requireH5Candidate('final-b2', slowInvocation, slowInvocation), /every original SLA/); +}); diff --git a/scripts/native-ingestion-performance-gates.test.mjs b/scripts/native-ingestion-performance-gates.test.mjs index 21753ab312c..281b16a9a52 100644 --- a/scripts/native-ingestion-performance-gates.test.mjs +++ b/scripts/native-ingestion-performance-gates.test.mjs @@ -1,10 +1,12 @@ import assert from "node:assert/strict"; +import { H5_MODE, H5_PROTOCOL, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, originalTerminal } from "../tests/e2e/fixtures/native-ingestion-h5.mjs"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; +import { gzipSync } from "node:zlib"; import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; test("four pairs must individually improve beyond noise", () => { @@ -161,6 +163,196 @@ function browserReport(revision, latency) { }) }; } +function h5Report(revision, loadedLatency) { + const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000); + Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE, + workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true, + actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture", + serverEntrySha256: "e".repeat(64), compiledHotPathHashes: { + "server/dist/services/heartbeat-run-events.js": "f".repeat(64), + "server/dist/services/native-runtime/paperclip-control-plane-port.js": "1".repeat(64), + "server/dist/vendor/paperclip-runner/native-session-runtime.js": "2".repeat(64) } }); + for (const item of report.cases) { + const latency = item.name === "burst512" ? loadedLatency : 1000; + item.settledAtMs = item.epochMs + latency; item.drainThroughTerminalMs = latency; + item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency }; + item.intendedToVisible = { p95Ms: latency }; + if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1; + item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs })); + const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs; + const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null, + startedAtMs: item.epochMs, deadlineMs: item.epochMs + 120000, assessedAtMs, + target: item.stop ? 3 : item.events, lastObservedCount: missed ? 30 : item.visible.length, + lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs, + error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null }, + observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } }; + const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000, + assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs, + lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false }; + item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation), + visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length, + independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10, + deltaCount: item.runs * (item.stop ? 3 : item.events), resultAcceptedCount: item.stop ? 0 : item.runs, clockStreamsClosed: true } }; + item.terminalStatuses = item.runIds.map(runId => ({ runId, runtimeMode: "native", status: item.stop ? "cancelled" : "succeeded" })); + for (const row of item.durable) row.resultAcceptedEvents = item.stop ? 0 : 1; + } + return report; +} + +test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires complete owning evidence", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-")); + const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url); + const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; }; + const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" }); + const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map(); + async function inventory(report, playwrightFile, name) { + const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision]; + if (!inventories.has(variant)) { + const roots = ["server/dist", "ui/dist", "packages/db/src", "packages/shared/src"]; + const files = Object.entries({ ...report.compiledHotPathHashes, "server/dist/index.js": report.serverEntrySha256, + "packages/db/src/client.ts": "3".repeat(64), "packages/shared/src/index.ts": "4".repeat(64), "ui/dist/index.html": "5".repeat(64) }) + .map(([path, sha256]) => ({ bytes: 10, path, sha256 })).sort((left, right) => left.path.localeCompare(right.path, "en")); + const aggregateSha256 = hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join("\n") + "\n"); + const manifestBytes = gzipSync(Buffer.from(JSON.stringify({ variant, revision: report.sourceRevision, + compiledBuildRevision: report.compiledBuildRevision, roots, fileCount: files.length, files, aggregateSha256 }))); + const manifestFile = path.join(directory, `${variant}-artifact-inventory.json.gz`); await writeFile(manifestFile, manifestBytes); + const manifestFileSha256 = hash(manifestBytes); + const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: { + manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } }); + const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision, + compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40), + binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }], + artifactInventorySummary: inventorySummaryFile }); + inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile, + inventorySummarySha256: hash(await readFile(inventorySummaryFile)), qualifiedReceiptFile, + qualifiedReceiptSha256: hash(await readFile(qualifiedReceiptFile)) }); + } + return save(`${name}-inventory.json`, { schema: "paperclip.native-ingestion.arm-inventory.v1", variant, + sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision, + playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) }); + } + async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) { + const entries = []; + for (let index = 0; index < 5; index++) { + const entry = {}; + for (const arm of ["baseline", "candidate"]) { + const report = h5Report(arm === "baseline" || sameRevision ? baselineRevision : candidateRevision, arm === "baseline" ? 180000 : candidateLatency); + const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0, + duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] }; + if (index === 0 && arm === "candidate") mutate?.(report, owner); + owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json", + body: Buffer.from(JSON.stringify(report)).toString("base64") }); + entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report); + entry[`${arm}PlaywrightFile`] = await save(`${prefix}-${index}-${arm}-playwright.json`, { suites: [{ tests: [owner] }] }); + entry[`${arm}InventoryFile`] = await inventory(report, entry[`${arm}PlaywrightFile`], `${prefix}-${index}-${arm}`); + } + entries.push(entry); + } + return save(`${prefix}-pairs.json`, entries); + } + try { + const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-")); + proof.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream"; + proof.measurementSha256 = "5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b"; + proof.harnessProvenance = { sourceRevision: "7d7e7a88d562a95a1d407b76474763b666c181f5", sourceStatus: "" }; + for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field]; + for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; } + const originalProof = await save("original-h4c-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{ + status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{ + name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] }); + const originalInventory = await inventory(proof, originalProof, "original-h4c"); + const finalProofArgs = [`--original-h4c-proof=${originalProof}`, `--original-h4c-inventory=${originalInventory}`]; + const noise = path.join(directory, "aa.json"); + const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]); + assert.equal(aa.status, 0, aa.stderr); + const aaReport = JSON.parse(await readFile(noise, "utf8")); + assert.equal(aaReport.h5Assessments.length, 5); + assert.deepEqual(aaReport.h5Assessments[0].baseline.misses, ["burst512:streaming120"]); + const cursor = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor", 180000, { candidateRevision: "B1" })}`, `--noise-report=${noise}`, "--require-acceptance"]); + assert.equal(cursor.status, 0, cursor.stderr); + const cursor900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor-matched900", 180000, { candidateRevision: "B1", duration: 950000 })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.equal(cursor900.status, 0, cursor900.stderr); + const added900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor-added900", 180000, { candidateRevision: "B1", mutate: (_report, owner) => { owner.results[0].duration = 950000; } })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(added900.status, 0, "Cursor candidate cannot add an unmatched900 miss"); + const goodPairs = await manifest("good", 1000); + const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]); + assert.equal(good.status, 0, good.stderr); + const wrongPolicy = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", `--pairs=${goodPairs}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(wrongPolicy.status, 0); assert.match(wrongPolicy.stderr, /Cursor policy requires B0→B1/); + const b1Noise = path.join(directory, "aa-b1.json"); + const b1AA = run(["--phase=aa", `--pairs=${await manifest("aa-b1", 181000, { sameRevision: true, baselineRevision: "B1" })}`, `--output=${b1Noise}`]); + assert.equal(b1AA.status, 0, b1AA.stderr); + const incremental = run(["--candidate-policy=final-b2", `--pairs=${await manifest("b1-b2", 1000, { baselineRevision: "B1" })}`, + `--noise-report=${b1Noise}`, ...finalProofArgs, "--require-acceptance"]); + assert.equal(incremental.status, 0, incremental.stderr); + const pendingFile = path.join(directory, "pending-h4c.json"); + const pending = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, `--output=${pendingFile}`, "--require-acceptance"]); + assert.notEqual(pending.status, 0); + const pendingReport = JSON.parse(await readFile(pendingFile, "utf8")); + assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false); + const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]); + assert.equal(performanceOnly.status, 0, performanceOnly.stderr); + for (const [name, change, message] of [ + ["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/], + ["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/], + ["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /unchanged original H4c/], + ]) { + const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0], + value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString()); + change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64"); + const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`); + const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, + `--original-h4c-proof=${file}`, `--original-h4c-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]); + assert.notEqual(attempt.status, 0, name); + const rejected = JSON.parse(await readFile(output, "utf8")); + assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false); + assert.match(rejected.originalH4cConfirmation.reason, message); + } + const badOutput = path.join(directory, "still-misses.json"); + const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`, + `--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]); + assert.notEqual(bad.status, 0); + const rejected = JSON.parse(await readFile(badOutput, "utf8")); + assert.equal(rejected.acceptancePassed, false); + assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false); + for (const [name, mutate, message] of [ + ["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/], + ["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/], + ["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/], + ["missing-delta", report => { report.cases[3].durable[99].individualDeltas--; }, /individualDeltas|result.events|512/], + ["wrong-workload", report => { report.workloadSha256 = "old"; }, /registration changed/], + ["pw-retry", (_report, owner) => { owner.results[0].retry = 1; }, /Retries/], + ["pw-error", (_report, owner) => { owner.results[0].errors.push({ message: "unexpected API failure" }); }, /deep-equal/], + ["wrong-observation-timeout", (_report, owner) => { owner.timeout = 900000; }, /1200000/], + ]) { + const result = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutate })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(result.status, 0, name); assert.match(result.stderr, message, name); + } + const outer = run(["--candidate-policy=final-b2", `--pairs=${await manifest("outer900", 1000, { duration: 950000 })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(outer.status, 0, "Observed total duration900 miss cannot qualify final B2"); + const incomplete = JSON.parse(await readFile(await manifest("missing-owner", 1000), "utf8")); + delete incomplete[0].candidatePlaywrightFile; + const missing = run(["--candidate-policy=final-b2", `--pairs=${await save("missing-owner-manifest.json", incomplete)}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(missing.status, 0); assert.match(missing.stderr, /independent owning Playwright/); + const global = JSON.parse(await readFile(await manifest("global-error", 1000), "utf8")); + const globalOwner = JSON.parse(await readFile(global[0].candidatePlaywrightFile, "utf8")); + globalOwner.errors = [{ message: "global teardown failed" }]; + await writeFile(global[0].candidatePlaywrightFile, JSON.stringify(globalOwner)); + const globalFailed = run(["--candidate-policy=final-b2", `--pairs=${await save("global-error-manifest.json", global)}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(globalFailed.status, 0); assert.match(globalFailed.stderr, /global error/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => { const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-")); const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url); diff --git a/tests/e2e/fixtures/native-ingestion-h5.md b/tests/e2e/fixtures/native-ingestion-h5.md new file mode 100644 index 00000000000..064ac26f95f --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-h5.md @@ -0,0 +1,155 @@ +# H5 complete observation registration (prospective; no execution release) + +H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5` +(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`). +Its observed 120-second streaming failures remain failures. Aborted or profiled +attempts cannot be reused as complete timing arms. H5 repairs observation of the +final outcome; it does not optimize the product or turn those failures into passes. + +The fresh application baselines remain B0 `a36567f9fbf805ad72207d54f539516e3121741e` +and B1 `d8cd3be0aeed4f9fb93ecc3cf725f1d46d5a0310`. Current B2 +`914129afb141d81c0d6d8c5f0ea11c20bd69a3da` has an observed original120s failure; +it is not already qualified. A later candidate uses its exact new immutable +revision, complete build receipt and inventory; no old binary is presumed equal. + +Select H5 explicitly with +`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5-observe-original-sla`. +The default code path retains the H4c 120/180/900-second behavior. H5 emits a new +protocol version, hashes its added helper with the driver sources, and records a +separate registration/workload SHA. The original comparator mode rejects H5. + +Run `node scripts/describe-native-ingestion-h5.mjs` to inspect the registration. +This is a dry run only: it starts no app, provider, database or browser and +accepts no execution arguments. It grants no runtime release. + +## Fixed work and observation boundaries + +Keep the five streaming cases and original local Stop case exactly as H4c: +paced 1×128 at25ms; burst 1×16; burst10×128; burst100×512; existing-history +1000 foreground prelude deltas plus measured10×128; local Stop1×512 at100ms, +clicked after at least three observed markers. Keep normal API/UI activation, +waves of two, shared300s readiness, actual GET+clock hellos, verified TLS, +closed READ ONLY REPEATABLE READ authority snapshot, source identities and +payloads, original nonawaited source-clock writes, and the existing2RAF observer. +No normalized-event polling channel, per-event HTTP, injected provider receipt, +raw recovery-policy change, diagnostic instrumentation or replacement app entry +is added. Fake-plugin streaming remains separate from real-cloud Stop proof. + +| Boundary | Original requirement | H5 observation | +| --- | --- | --- | +| Foreground markers | Original Playwright120s wait | Continue to an absolute600s ceiling from that same wait start | +| All terminal runs |180s after foreground completion (and Stop click) | Same180s after eventual foreground completion; hard failure at the ceiling | +| Stop acknowledgement and leases | Existing30s polls and ordering | Unchanged; full click→acknowledgement metric retained | +| Invocation | Original900s test budget | Actual1200s observation timeout, separately disclosed | + +The first120s poll is the original matcher. Only that exact known matcher +deadline can extend observation. Page/API errors, other assertions, terminal +failures and cleanup failures never become an allowed SLA miss. There is no +collection wait when markers are available. The600s ceiling is absolute, so a +120s miss leaves at most480s additional observation, not another600s. + +Record the original wait start/deadline, actual last count and its observation +time, assessment time and error before continuing. Keep the original failure +after completion. The original120s clock starts before the common source epoch; +it is not silently rebased to epoch+120s. Derive a separate paint-deadline count +from retained2RAF timestamps; do not mislabel it as the original poll's count. + +If120s fails, original terminal processing was **not reached**. Later180s +terminal observations are separately labeled and cannot make the original +protocol pass. Keep epoch→all-terminal drainage unshortened. An additional +independent completion upper bound follows existing screenshot, reload, +persistence, clock and lease checks; it includes verification overhead and is +not an exact last-commit time. + +The owning public Playwright JSON result supplies duration, startTime, retry, +status, errors and configured timeout. Its observed total duration is an +independent conservative assessment of the original900s budget; hooks/teardown +can be treated differently by the original timeout. It is not an exact original +timer-expiration observation. A final candidate must have total duration<900s +in every arm and later pass one unchanged original H4c full invocation on the +same final SUT revision and full qualified artifact inventory. + +## Complete evidence and eligibility + +H5 `observationComplete` means all six cases reached finite correct outcomes, +with every expected participant and foreground marker. It does not mean H4c +passed. Every remote run must succeed in native mode, retain every normalized +delta's complete payload/identity/order, have exactly one independently read +`run.result.accepted`, and pass the existing reload/result/clock/lease checks. +The100-run case must independently verify all51200 deltas. Stop keeps its +original cancellation, durable acknowledgement, local journal, PID/group/provider +absence and lease checks; it must not acquire an accepted success result. + +An observation ceiling, failed/cancelled remote run, missing event/result, +unexpected API error, incomplete source clock, replay conflict or cleanup error +is ineligible. Preserve its raw partial evidence and censoring state. Never +substitute the ceiling as latency, compute a512-event tail from a visible prefix, +discard a slow arm, retry it into the sample, cancel background work to make a +pair finish, or hide work remaining after the measured interval. + +Each comparison-manifest entry contains `baselineFile`, `candidateFile`, +`baselinePlaywrightFile`, `candidatePlaywrightFile`, `baselineInventoryFile`, and +`candidateInventoryFile`. The owning Playwright result must contain exactly that +evidence attachment, one passed observation attempt, retry0, no errors, and the +actual1200s timeout. An inventory sidecar binds that report hash to the exact +gzip inventory bytes, full descriptor aggregate, summary and qualified build +receipt hashes, source/build revision, and reported serving/native fingerprints. +These frozen artifacts are evidence of the qualified build; the external owner +still checks the live checkout/build inventories before each arm. A sidecar does +not itself attest that the live filesystem stayed unchanged. + +| Role | Allowed original failures | Required result | +| --- | --- | --- | +| Fresh B0/B1 A/A arms | Known burst512 streaming120 miss; explicit conservative observed-duration900 miss | Complete finite correct observation; root-reviewed noise suitability | +| B0→B1 cursor-only candidate | Only an exact registered known miss also present in the paired baseline: burst512 streaming120 or conservative observed-duration900 | No additional deadline failures; latency/control nonregression plus separate incremental SQL evidence; never claim SLA fixed | +| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later unchanged H4c proof | + +Only burst512's streaming miss is registered as a known workload SLA failure. +New failures in other workloads require diagnosis. The existing Stop poll +ordering is preserved; no invented absolute click→ack30s requirement replaces it. + +## Stage order and acceptance + +Prospectively freeze **B0 A/A → B1 A/A → B0→B1 nonregression → B1→B2 → B0→B2**. +Each stage has exactly five fresh alternating pairs, separate disposable state +per arm, matched payloads/load/resource/cache policy, no reused baseline arms, +and no concurrent builds/profilers/benchmarks on the measurement VM. Root reviews +and freezes both complete A/A envelopes before any further product fix. Local +product implementation can then proceed while the unchanged B0→B1 stage runs; +the candidate VM build waits until that stage is quiet. + +Keep the existing per-metric maximum absolute paired A/A envelope, median gate, +and at least4/5 individual noise-exceeding loaded wins. Each loaded p95 +emission-to-visible, p95 intended-to-visible and terminal-drainage metric must +improve; other fixed latency/control metrics must not regress outside noise. +Do not infer p99 from sparse observations. There is no new fixed percentage A/A +cutoff: root must reject unsuitable noisy/nonstationary calibration explicitly. + +Use `--observation-mode=h5-observe-original-sla` for the comparator. Candidate +comparisons also require `--candidate-policy=cursor-nonregression --expect=nonregression` +or `--candidate-policy=final-b2`. Final comparisons expose `performancePassed` +separately; `acceptancePassed` remains false until `--original-h4c-proof` and +`--original-h4c-inventory` establish the later fresh original functional pass. +That proof is full, unprofiled, retry0, original900s, on the exact final SUT +revision/qualified inventory, with a new disposable company and a start time +after all candidate arms. Prior failed H4c attempts remain in the record. +Use the explicit H5 `--require-performance` gate for the two final comparison +stages while that later proof is pending; this requires every performance and +candidate original-SLA gate, but grants no final acceptance. After the one later +original H4c qualification, recompute both reports against their unchanged paired +manifests with the proof and `--require-acceptance`. This re-evaluates retained +evidence; it neither reuses an arm in a different comparison nor reruns failures. + +Before any runtime release, review the exact H5 protocol/comparator/helper +hashes, registration dry run, outcome/eligibility tests, types and test listing. +Then qualify the changed observation path before fresh A/A. Each later stage +requires a separate root release pinning exact driver, build/inventory and +reviewed noise-report bytes; verify the noise hash before the first arm and +again before comparison. Product or workload changes require new prospective +provenance and affected requalification; do not loosen gates after results. + +Separate successful full-window diagnostics may later assess Node/PG CPU, +queue/backlog, delay and memory with identical capture hooks. Aborted profiles +remain diagnostic-only and cannot establish complete CPU/event costs. No CPU +requirement or profiling is added to this timing comparator. Local results do +not establish deployed improvement. diff --git a/tests/e2e/fixtures/native-ingestion-h5.mjs b/tests/e2e/fixtures/native-ingestion-h5.mjs new file mode 100644 index 00000000000..a38a5797da1 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-h5.mjs @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; + +export const H5_MODE = 'h5-observe-original-sla'; +export const H5_PROTOCOL = 'v5-complete-observation-original-sla'; +export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'a36567f9fbf805ad72207d54f539516e3121741e', + B1: 'd8cd3be0aeed4f9fb93ecc3cf725f1d46d5a0310' }); +export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000, + streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }); +export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 }); +export const H5_WORKLOADS = Object.freeze([ + { name: 'paced', runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false }, + { name: 'burst16', runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false }, + { name: 'burst128', runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false }, + { name: 'burst512', runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false }, + { name: 'existing-history', runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false }, + { name: 'operator-stop', runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true }, +]); +export const H5_REGISTRATION = Object.freeze({ mode: H5_MODE, protocolVersion: H5_PROTOCOL, + baselineRevisions: H5_BASELINE_REVISIONS, + originalBudgets: ORIGINAL_BUDGETS, observationLimits: H5_LIMITS, workloads: H5_WORKLOADS, + pairs: 5, order: 'baseline-first on even pair index; candidate-first on odd pair index', + stages: ['B0 A/A', 'B1 A/A', 'B0→B1 nonregression', 'B1→B2 improvement', 'B0→B2 improvement'], + knownBaselineMisses: ['burst512:streaming120', 'invocation:observed-total-duration900'], + baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of H4c', + cursorCandidateEligibility: 'Nonregression; only the paired baseline exact registered known misses may be inherited', + finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later unchanged H4c full functional PASS required', + noise: 'Fresh B0 and B1 A/A frozen before any further product implementation; no old abort/profile samples', + performance: 'Loaded p95 emission/intended-to-visible and terminal drainage improve beyond frozen noise in median and at least4/5 pairs; all other metrics nonregress', +}); +export const H5_WORKLOAD_SHA256 = createHash('sha256').update(JSON.stringify(H5_REGISTRATION)).digest('hex'); + +export function h5Enabled(value) { + assert.ok(value === undefined || value === H5_MODE, 'Explicit supported H5 observation mode required'); + return value === H5_MODE; +} + +// Keep the original Playwright poll as the first wait. Only its known matcher +// deadline may extend observation; callback errors and other failures propagate. +export async function observeStreaming({ readCount, poll, target, now = Date.now, onOutcome = _value => {} }) { + const startedAtMs = now(); + let lastObservedCount = null, lastObservedAtMs = null, readError; + const read = async () => { + try { + const count = await readCount(); + assert.ok(Number.isSafeInteger(count) && count >= 0, 'Invalid visible marker count'); + lastObservedCount = count; lastObservedAtMs = now(); return count; + } catch (error) { readError = error; throw error; } + }; + let deadlineError; + try { await poll(read, ORIGINAL_BUDGETS.streamingMs, target); } + catch (error) { + if (readError) throw readError; + assert.ok(error?.matcherResult?.name === 'toBeGreaterThanOrEqual' && error.matcherResult.pass === false + && error.message.includes(`Timeout ${ORIGINAL_BUDGETS.streamingMs}ms exceeded while waiting on the predicate`), + 'Only the original streaming matcher deadline can extend observation'); + assert.ok(lastObservedCount !== null && lastObservedCount < target, 'Deadline needs an incomplete actual observation'); + deadlineError = error; + } + if (readError) throw readError; + const original = { status: deadlineError ? 'failed' : 'passed', reason: deadlineError ? 'original_streaming_deadline' : null, + startedAtMs, deadlineMs: startedAtMs + ORIGINAL_BUDGETS.streamingMs, assessedAtMs: now(), + target, lastObservedCount, lastObservedAtMs, error: deadlineError?.message ?? null }; + const observation = { ceilingMs: H5_LIMITS.streamingMs, deadlineMs: startedAtMs + H5_LIMITS.streamingMs, + complete: false, censored: false, completedAtMs: null }; + const result = { original, observation }; + onOutcome(result); + if (deadlineError) { + try { + const remaining = observation.deadlineMs - now(); + assert.ok(remaining > 0, 'H5 streaming observation ceiling exceeded'); + await poll(read, remaining, target); + } catch (error) { observation.censored = true; onOutcome(result); throw readError ?? error; } + } + if (readError) throw readError; + observation.complete = true; observation.completedAtMs = now(); onOutcome(result); + return result; +} + +export function originalTerminal(streaming, terminal) { + return streaming.original.status === 'passed' + ? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null : 'original_terminal_deadline', + startedAtMs: terminal.startedAtMs, deadlineMs: terminal.deadlineMs, assessedAtMs: terminal.assessedAtMs } + : { status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null }; +} + +export function assessH5Arm(report, owner) { + assert.equal(report.protocolVersion, H5_PROTOCOL); + assert.equal(report.observationMode, H5_MODE); + assert.equal(report.observationComplete, true, 'H5 requires complete finite observation'); + assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false); + assert.equal(report.diagnosticOnly, false); assert.equal(report.performanceAcceptanceEligible, true); + assert.equal(report.cleanup?.complete, true); assert.deepEqual(report.cleanup.errors, []); + assert.equal(report.workloadSha256, H5_WORKLOAD_SHA256, 'H5 registration changed'); + assert.deepEqual(report.observationLimits, H5_LIMITS); + assert.deepEqual(report.resourceBudgets, ORIGINAL_BUDGETS); + assert.equal(report.actualPlaywrightTimeoutMs, H5_LIMITS.fullInvocationMs); + assert.deepEqual(report.cases.map(item => item.name), H5_WORKLOADS.map(item => item.name), 'Complete fixed six-case observation required'); + assert.equal(owner.status, 'passed', 'Owning Playwright attempt must pass observation/correctness'); + assert.equal(owner.retry, 0, 'Retries cannot replace an H5 arm'); + assert.deepEqual(owner.errors, []); + assert.equal(owner.timeoutMs, H5_LIMITS.fullInvocationMs); + assert.ok(Number.isFinite(owner.durationMs) && owner.durationMs > 0 && owner.durationMs < H5_LIMITS.fullInvocationMs); + assert.ok(typeof owner.startedAt === 'string' && Number.isFinite(Date.parse(owner.startedAt))); + const misses = []; + for (const item of report.cases) { + const value = item.h5; + assert.ok(value, 'Missing H5 case outcomes'); + assert.equal(value.streaming.observation.complete, true); + assert.equal(value.streaming.observation.censored, false); + const streaming = value.streaming.original; + for (const field of ['startedAtMs', 'deadlineMs', 'assessedAtMs', 'lastObservedAtMs']) + assert.ok(Number.isSafeInteger(streaming[field]) && streaming[field] > 0, 'Missing raw original-deadline timing'); + assert.ok(['passed', 'failed'].includes(streaming.status)); + assert.equal(streaming.deadlineMs, streaming.startedAtMs + ORIGINAL_BUDGETS.streamingMs); + assert.equal(streaming.target, item.stop ? 3 : item.events); + assert.ok(Number.isSafeInteger(streaming.lastObservedCount) && streaming.lastObservedCount >= 0); + assert.ok(streaming.lastObservedAtMs >= streaming.startedAtMs && streaming.lastObservedAtMs <= streaming.assessedAtMs); + if (streaming.status === 'failed') { + assert.equal(streaming.reason, 'original_streaming_deadline'); + assert.ok(streaming.lastObservedCount < streaming.target); + assert.ok(streaming.assessedAtMs >= streaming.deadlineMs); + assert.equal(item.stop, false, 'Control readiness cannot inherit a streaming SLA miss'); + assert.ok(streaming.error.includes('Timeout 120000ms exceeded while waiting on the predicate')); + assert.equal(value.terminalOriginal.status, 'not-reached'); + assert.equal(value.terminalOriginal.reason, 'original_streaming_deadline'); + misses.push(`${item.name}:streaming120`); + } else { + assert.ok(streaming.lastObservedCount >= streaming.target); + assert.equal(streaming.reason, null); + assert.equal(streaming.error, null); + assert.equal(value.terminalOriginal.status, 'passed'); + } + assert.equal(value.terminalObservation.complete, true, 'H5 terminal observation must drain'); + assert.equal(value.terminalObservation.censored, false); + assert.equal(value.terminalObservation.deadlineMs, value.terminalObservation.startedAtMs + H5_LIMITS.terminalDrainageMs); + assert.equal(value.terminalObservation.assessedAtMs, item.settledAtMs); + assert.equal(value.terminalObservation.observedRuns, item.runs); assert.equal(value.terminalObservation.terminalRuns, item.runs); + assert.ok(value.terminalObservation.lastObservedAtMs >= value.terminalObservation.startedAtMs + && value.terminalObservation.lastObservedAtMs <= value.terminalObservation.assessedAtMs); + assert.deepEqual(value.terminalObservation.lastStatuses.map(row => row.runId).sort(), [...item.runIds].sort()); + assert.ok(value.terminalObservation.lastStatuses.every(row => row.status === (item.stop ? 'cancelled' : 'succeeded'))); + assert.equal(value.streaming.observation.ceilingMs, H5_LIMITS.streamingMs); + assert.equal(value.streaming.observation.deadlineMs, streaming.startedAtMs + H5_LIMITS.streamingMs); + assert.ok(value.streaming.observation.completedAtMs >= streaming.assessedAtMs + && value.streaming.observation.completedAtMs <= value.streaming.observation.deadlineMs); + assert.ok(value.terminalObservation.startedAtMs >= value.streaming.observation.completedAtMs); + assert.ok(item.settledAtMs <= value.terminalObservation.deadlineMs); + assert.deepEqual(value.terminalOriginal, originalTerminal(value.streaming, value.terminalObservation)); + assert.equal(value.visibleCountByOriginalPaintDeadline, item.visible.filter(marker => marker.visibleMs <= streaming.deadlineMs).length); + assert.deepEqual(item.terminalStatuses.map(row => row.runId).sort(), [...item.runIds].sort()); + assert.ok(item.terminalStatuses.every(row => row.runtimeMode === 'native' && row.status === (item.stop ? 'cancelled' : 'succeeded')), + 'Failed terminal runs are not successful drainage'); + assert.ok(item.durable.every(row => row.resultAcceptedEvents === (item.stop ? 0 : 1)), 'Every completed run needs its independently persisted result'); + assert.ok(value.independentDrain.verifiedAtMs >= item.settledAtMs); + assert.equal(value.independentDrain.elapsedFromEpochMs, value.independentDrain.verifiedAtMs - item.epochMs); + assert.equal(value.independentDrain.deltaCount, item.durable.reduce((sum, row) => sum + row.individualDeltas, 0)); + assert.equal(value.independentDrain.resultAcceptedCount, item.stop ? 0 : item.runs); + assert.equal(value.independentDrain.clockStreamsClosed, true); + } + if (owner.durationMs >= ORIGINAL_BUDGETS.fullInvocationMs) misses.push('invocation:observed-total-duration900'); + return { originalSlaPassed: misses.length === 0, misses, + original900budget: { status: owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs ? 'passed' : 'failed', + assessedAgainst: 'observed owning Playwright total duration', durationMs: owner.durationMs, + limitMs: ORIGINAL_BUDGETS.fullInvocationMs, + scope: 'Independent conservative gate; total duration can include hooks/teardown that the original timeout treats differently; not an exact original timer-expiration fact' } }; +} + +export function requireH5Candidate(policy, baseline, candidate) { + if (policy === 'final-b2') assert.deepEqual(candidate.misses, [], 'Final B2 must pass every original SLA in every arm'); + else if (policy === 'cursor-nonregression') { + const allowed = new Set(H5_REGISTRATION.knownBaselineMisses); + assert.ok(candidate.misses.every(miss => allowed.has(miss) && baseline.misses.includes(miss)), + 'Cursor-only candidate may inherit only the paired baseline exact registered known misses'); + } else throw new Error('Explicit final-b2 or cursor-nonregression candidate policy required'); +} diff --git a/tests/e2e/native-event-ingestion.journey.ts b/tests/e2e/native-event-ingestion.journey.ts index bce7653d9db..853549253db 100644 --- a/tests/e2e/native-event-ingestion.journey.ts +++ b/tests/e2e/native-event-ingestion.journey.ts @@ -7,6 +7,8 @@ import { hostname } from "node:os"; import path from "node:path"; import { expect, test } from "@playwright/test"; import { json } from "./agent-chat.shared"; +import { h5Enabled, H5_MODE, H5_PROTOCOL, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS, + observeStreaming, originalTerminal } from "./fixtures/native-ingestion-h5.mjs"; type SeenMarker = { markerCode: string; index: number; visibleMs: number }; type Emission = { nonce: string; markerCode: string; index: number; intendedMs: number; emittedMs: number; receivedAtMs: number }; @@ -23,6 +25,7 @@ type SetupActivation = { index: number; issueId: string; token: string; waveInde type SetupWave = { index: number; indices: number[]; startedAtMs: number; settledAtMs?: number; statuses?: Array<"fulfilled" | "rejected"> }; const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1"; +const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE); const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "sandbox-remote-with-local-stop"; const providerReadinessMs = 300_000; const setupActivationConcurrency = 2; @@ -150,6 +153,7 @@ const processAbsent = (pid: number) => { }; test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => { + const bodyStartedAtMs = Date.now(); const pending = new Map>(); const clocks = new Map(); let markerCounter = 0; @@ -271,10 +275,13 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video), screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" }; const report: Record = { - protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport, + protocolVersion: h5 ? H5_PROTOCOL : "v4-remote-streaming-original-local-stop-clock-stream", transport, + ...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, + observationComplete: false, bodyStartedAtMs, actualPlaywrightTimeoutMs: info.timeout, + originalInvocationBudgetAssessment: "Requires independent owning Playwright observed total duration; conservative original900 assessment, not exact timeout expiration" } : {}), transportPolicy: { streaming: "sandbox-remote", operatorStop: "local", realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, - resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000, + resourceBudgets: h5 ? ORIGINAL_BUDGETS : { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }, admissionProtocol: "two-at-a-time-provider-and-clock-readiness-before-common-epoch", setupActivationConcurrency, @@ -307,7 +314,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex"); const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs", "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs", - "fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs"]; + "fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs", + ...(h5 ? ["fixtures/native-ingestion-h5.mjs"] : [])]; const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) }))); report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex"); const harnessRoot = path.resolve(import.meta.dirname, "../.."); @@ -529,7 +537,12 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy active.participants = participants; for (const response of pending.get(key)!.values()) response.writeHead(200, { "Content-Type": "application/json" }).end(JSON.stringify({ epochMs })); const observed = () => page.evaluate(code => (window as unknown as { nativeIngestionVisible: SeenMarker[] }).nativeIngestionVisible.filter(item => item.markerCode === code), markerCode); - await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events); + let streamingObservation: Awaited> | undefined; + if (h5) streamingObservation = await observeStreaming({ readCount: async () => (await observed()).length, + target: stop ? 3 : workload.events, + poll: (read: () => Promise, timeout: number, target: number) => expect.poll(read, { timeout }).toBeGreaterThanOrEqual(target), + onOutcome: (value: unknown) => { active.streamingObservation = value; } }); + else await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events); let stopRequestedAt: number | undefined; if (stop) { const button = page.getByRole("button", { name: "Stop", exact: true }); @@ -539,11 +552,25 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy } active.phase = "terminal-drainage"; let runs: any[] = []; - await expect.poll(async () => { + const terminalStartedAtMs = h5 ? Date.now() : undefined; + const terminalObservation = h5 ? { startedAtMs: terminalStartedAtMs!, deadlineMs: terminalStartedAtMs! + H5_LIMITS.terminalDrainageMs, + assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0, + lastStatuses: [] as Array<{ runId: string; status: string }>, complete: false, censored: false } : undefined; + if (h5) active.terminalObservation = terminalObservation; + try { await expect.poll(async () => { runs = (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId)); + if (terminalObservation) Object.assign(terminalObservation, { lastObservedAtMs: Date.now(), observedRuns: runs.length, + terminalRuns: runs.filter(run => ["succeeded", "cancelled", "failed"].includes(run.status)).length, + lastStatuses: runs.map(run => ({ runId: run.id, status: run.status })) }); return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status)); - }, { timeout: 180_000 }).toBe(true); + }, { timeout: 180_000 }).toBe(true); } + catch (error) { + if (terminalObservation) { terminalObservation.assessedAtMs = Date.now(); terminalObservation.censored = true; + active.terminalOriginal = originalTerminal(streamingObservation!, terminalObservation); } + throw error; + } const settledAt = Date.now(), seenMarkers = await observed(); + if (terminalObservation) { terminalObservation.assessedAtMs = settledAt; terminalObservation.complete = true; } active.settledAt = settledAt; active.seenMarkers = seenMarkers; const details = []; for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`))); @@ -618,6 +645,11 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs }; const clockEvidence: Array> = []; const result = { ...workload, stop, transport: caseTransport, environmentId: caseEnvironment.id, + ...(h5 ? { h5: { streaming: streamingObservation, terminalObservation, + terminalOriginal: originalTerminal(streamingObservation!, terminalObservation!), + visibleCountByOriginalPaintDeadline: seenMarkers.filter(marker => marker.visibleMs <= streamingObservation!.original.deadlineMs).length, + deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : {}), + ...(h5 ? { terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status })) } : {}), setupActivationConcurrency, setupReadinessStartedAtMs, readinessDeadlineMs, allReadyObservedAtMs, activations, activationWaves, nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt, stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps, @@ -648,16 +680,19 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const code = state.streams.get(token)!.markerCode; expect(token).toBeTruthy(); const deltas: Array<{ index: number; identity: string }> = []; - let afterSeq = 0; + let afterSeq = 0, resultAcceptedEvents = 0; while (true) { const rows = await json(await request.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); if (!rows.length) break; for (const row of rows) { const event = row.payload?.prpEvent; + if (h5 && (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted")) resultAcceptedEvents++; if (event?.eventType !== "item.delta") continue; for (const match of String(event.payload?.text ?? "").matchAll(new RegExp(`\\[${code}([a-z0-9]{1,3})\\]`, "g"))) { const index = Number.parseInt(match[1], 36); expect(event.payload.text, "Every durable normalized delta retains its complete compact payload").toBe(`[${code}${index.toString(36)}] `); + if (h5) { expect(event.runId).toBe(run.id); expect(event.sourceInstanceId).toBeTruthy(); + expect(event.sourceEventId).toBeTruthy(); expect(Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > 0).toBe(true); } deltas.push({ index, identity: `${event.sourceInstanceId}:${event.sourceEventId}:${event.sourceSeq}` }); } } @@ -674,11 +709,21 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy expect(new Set(deltas.map(item => item.identity)).size).toBe(deltas.length); expect(deltas.map(item => item.index)).toEqual(Array.from({ length: stop ? deltas.length : workload.events }, (_, index) => index)); if (stop) for (const item of visible) expect(deltas.some(delta => delta.index === item.index)).toBe(true); - durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, matchedSourceClocks: deltas.length, ordered: true }); + if (h5) expect(resultAcceptedEvents).toBe(stop ? 0 : 1); + durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, + matchedSourceClocks: deltas.length, ordered: true, ...(h5 ? { resultAcceptedEvents } : {}) }); } await expect.poll(() => [...state.streams.values()].every(stream => stream.ended || stream.aborted), { timeout: 30_000 }).toBe(true); expect(state.errors).toEqual([]); expect([...state.streams.values()].every(stream => !stream.error)).toBe(true); + if (h5) { + const verifiedAtMs = Date.now(); + Object.assign(result.h5!, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs, + deltaCount: durable.reduce((count, row) => count + row.individualDeltas, 0), + resultAcceptedCount: durable.reduce((count, row) => count + (row.resultAcceptedEvents ?? 0), 0), + clockStreamsClosed: true, + scope: "Independent completion upper bound after existing reload/API/clock/lease checks; includes verification and screenshot overhead, not exact last-commit time" } }); + } clockEvidence.push(...[...state.streams.values()].map(stream => ({ nonce: stream.nonce, markerCode: stream.markerCode, launcher: stream.launcher, providerPid: stream.providerPid, hello: stream.hello, ended: stream.ended, aborted: stream.aborted, error: stream.error, partialBytes: stream.partialBytes, @@ -691,6 +736,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy for (const workload of cases) await perform(workload); await perform({ name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0 }, true); report.complete = true; + if (h5) report.observationComplete = true; } catch (error) { report.primaryFailure = { name: error instanceof Error ? error.name : "UnknownError", message: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined }; @@ -728,7 +774,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy await attemptCleanup("restore-experimental-settings", async () => json(await request.patch("/api/instance/settings/experimental", { data: { enableNativeRunner: original.enableNativeRunner, enableWorktreeRunExecution: original.enableWorktreeRunExecution } }))); report.cleanup = { errors: cleanupErrors, complete: cleanupErrors.length === 0 }; - if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; } + if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; + if (h5) report.observationComplete = false; } + if (h5) report.bodyFinishedAtMs = Date.now(); await info.attach("native-ingestion-evidence", { body: JSON.stringify(report, null, 2), contentType: "application/json" }); if (!report.primaryFailure) expect(cleanupErrors, "API cleanup must succeed before accepting an invocation").toEqual([]); } diff --git a/tests/e2e/playwright-native-ingestion.config.ts b/tests/e2e/playwright-native-ingestion.config.ts index 47c7ce3276f..083380b1ca5 100644 --- a/tests/e2e/playwright-native-ingestion.config.ts +++ b/tests/e2e/playwright-native-ingestion.config.ts @@ -3,9 +3,11 @@ import os from "node:os"; import path from "node:path"; import { defineConfig, type ReporterDescription } from "@playwright/test"; import base from "./playwright.config"; +import { h5Enabled, H5_LIMITS } from "./fixtures/native-ingestion-h5.mjs"; const server = base.webServer as Exclude; const evidence = process.env.PAPERCLIP_INGESTION_EVIDENCE_DIR; +const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE); const browserRecording = Boolean(process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR) || process.env.PAPERCLIP_INGESTION_BROWSER_CAPTURE === "1"; process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "off"; const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../..")); @@ -33,7 +35,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-" export default defineConfig({ ...base, testMatch: "native-event-ingestion.journey.ts", - timeout: 900_000, + timeout: h5 ? H5_LIMITS.fullInvocationMs : 900_000, ...(evidence ? { outputDir: path.join(evidence, "test-results"), reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}), use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 },