mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: package frozen native ingestion H5b tooling
Keep the executed H5b driver and helper provenance with the cursor PR. Record the failed full qualification and retain separate runtime inputs. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
664036ed7e
commit
652e4aebff
16 files changed
+2494
-134
No files matched your search
@@ -3,6 +3,9 @@ import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS,
|
||||
assessH5Arm, requireH5Candidate, assertParticipantSetup, assertParticipantTopology } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs";
|
||||
|
||||
// Each manifest entry points at native-ingestion-evidence JSON attachments
|
||||
// from two isolated compiled-app runs. Run the variants in alternating order;
|
||||
@@ -11,12 +14,88 @@ const arg = (name: string) => process.argv.find(value => value.startsWith(`--${n
|
||||
const manifestPath = arg("pairs");
|
||||
assert.ok(manifestPath, "Pass --pairs=<JSON manifest of baselineFile/candidateFile pairs>");
|
||||
const phase = arg("phase") ?? "compare", expectation = arg("expect") ?? "improvement";
|
||||
const h5 = arg("observation-mode") === H5_MODE;
|
||||
assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode");
|
||||
const candidatePolicy = arg("candidate-policy");
|
||||
assert.ok(!arg("original-h4c-proof") && !arg("original-h4c-inventory"), "Old H4c proof arguments cannot confirm the new independent-agent geometry");
|
||||
assert.ok(["aa", "compare"].includes(phase));
|
||||
assert.ok(["improvement", "nonregression"].includes(expectation));
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string }>;
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string;
|
||||
baselinePlaywrightFile?: string; candidatePlaywrightFile?: string; baselineInventoryFile?: string; candidateInventoryFile?: string }>;
|
||||
assert.ok(manifest.length >= 5, "At least five actual paired application runs are required");
|
||||
if (h5) assert.equal(manifest.length, 5, "H5 requires exactly five prospectively registered pairs");
|
||||
const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8"));
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) })));
|
||||
async function owningResult(reportPath: string, evidence?: any) {
|
||||
const bytes = await readFile(reportPath);
|
||||
const report = JSON.parse(bytes.toString("utf8")), tests: any[] = [];
|
||||
assert.deepEqual(report.errors ?? [], [], "Owning Playwright report has a global error");
|
||||
const visit = (value: any) => {
|
||||
if (Array.isArray(value)) for (const child of value) visit(child);
|
||||
else if (value && typeof value === "object") {
|
||||
if (Array.isArray(value.results) && Object.hasOwn(value, "expectedStatus")) tests.push(value);
|
||||
for (const [key, child] of Object.entries(value)) if (key !== "results") visit(child);
|
||||
}
|
||||
};
|
||||
visit(report); assert.equal(tests.length, 1, "One complete owning test is required; no partial selection");
|
||||
const test = tests[0]; assert.equal(test.results.length, 1, "No retry or omitted owning attempt");
|
||||
assert.equal(test.expectedStatus, "passed");
|
||||
const result = test.results[0], attachments = result.attachments.filter((item: any) => item.name === "native-ingestion-evidence");
|
||||
assert.ok(result.error === undefined || result.error === null, "Owning Playwright attempt has an error");
|
||||
assert.equal(attachments.length, 1, "Owning result must carry this exact evidence attachment");
|
||||
const attachment = attachments[0];
|
||||
const attached = attachment.body ? JSON.parse(Buffer.from(attachment.body, "base64").toString("utf8"))
|
||||
: JSON.parse(await readFile(path.resolve(path.dirname(reportPath), attachment.path), "utf8"));
|
||||
if (evidence) assert.deepEqual(attached, evidence, "Playwright result belongs to a different evidence arm");
|
||||
return { evidence: attached, owner: { reportFile: reportPath, reportSha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
status: result.status, retry: result.retry, errors: result.errors, startedAt: result.startTime,
|
||||
durationMs: result.duration, timeoutMs: test.timeout } };
|
||||
}
|
||||
async function loadArm(file: string, playwrightFile?: string, inventoryFile?: string) {
|
||||
const evidence = await load(file);
|
||||
if (!h5) return evidence;
|
||||
assert.ok(playwrightFile, "H5 requires each arm's independent owning Playwright JSON report");
|
||||
evidence.owningPlaywright = (await owningResult(path.resolve(path.dirname(manifestPath!), playwrightFile), evidence)).owner;
|
||||
assert.ok(inventoryFile, "H5 requires each arm's exact full artifact inventory sidecar");
|
||||
evidence.armInventory = await validateArmInventory(path.resolve(path.dirname(manifestPath!), inventoryFile),
|
||||
{ evidence, owningPlaywright: evidence.owningPlaywright });
|
||||
return evidence;
|
||||
}
|
||||
const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await loadArm(entry.baselineFile, entry.baselinePlaywrightFile, entry.baselineInventoryFile),
|
||||
candidate: await loadArm(entry.candidateFile, entry.candidatePlaywrightFile, entry.candidateInventoryFile) })));
|
||||
const h5Assessments: any[] = [];
|
||||
if (h5) {
|
||||
if (phase === "compare") {
|
||||
assert.ok(["final-b2", "cursor-nonregression"].includes(candidatePolicy ?? ""), "H5 requires an explicit candidate SLA policy");
|
||||
assert.equal(expectation, candidatePolicy === "cursor-nonregression" ? "nonregression" : "improvement");
|
||||
}
|
||||
for (const pair of pairs) {
|
||||
assert.ok(Object.values(H5_BASELINE_REVISIONS).includes(pair.baseline.sourceRevision), "H5 baseline must use the prospectively frozen B0 or B1 revision");
|
||||
const baselineVariant = pair.baseline.sourceRevision === H5_BASELINE_REVISIONS.B0 ? "B0" : "B1";
|
||||
assert.equal(pair.baseline.armInventory.variant, baselineVariant, "Baseline inventory role differs from its frozen revision");
|
||||
if (phase === "aa") assert.equal(pair.candidate.armInventory.variant, baselineVariant, "A/A must use the same baseline inventory role");
|
||||
if (phase === "compare" && candidatePolicy === "cursor-nonregression") {
|
||||
assert.equal(pair.baseline.sourceRevision, H5_BASELINE_REVISIONS.B0, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.sourceRevision, H5_BASELINE_REVISIONS.B1, "Cursor policy requires B0→B1");
|
||||
assert.equal(pair.candidate.armInventory.variant, "B1", "Cursor policy cannot stand in for final B2 acceptance");
|
||||
} else if (phase === "compare") {
|
||||
assert.equal(pair.candidate.armInventory.variant, "B2", "Final policy requires the qualified B2 inventory role");
|
||||
assert.ok(!Object.values(H5_BASELINE_REVISIONS).includes(pair.candidate.sourceRevision), "Final B2 must differ from both frozen baselines");
|
||||
}
|
||||
const baseline = assessH5Arm(pair.baseline, pair.baseline.owningPlaywright);
|
||||
const candidate = assessH5Arm(pair.candidate, pair.candidate.owningPlaywright);
|
||||
// The only prospectively known baseline workload SLA miss is loaded streaming.
|
||||
const allowed = new Set(["burst512:streaming120", "invocation:observed-total-duration900"]);
|
||||
for (const miss of baseline.misses) assert.ok(allowed.has(miss), "Unknown baseline deadline failure needs diagnosis");
|
||||
if (phase === "aa") for (const miss of candidate.misses) assert.ok(allowed.has(miss), "Unknown A/A deadline failure needs diagnosis");
|
||||
let candidateSlaPolicyError;
|
||||
if (phase === "compare") {
|
||||
try { requireH5Candidate(candidatePolicy, baseline, candidate); }
|
||||
catch (error) { candidateSlaPolicyError = error instanceof Error ? error.message : String(error); }
|
||||
}
|
||||
h5Assessments.push({ baseline, candidate, candidateSlaPolicyError, candidateSlaPolicyPassed: !candidateSlaPolicyError,
|
||||
owningPlaywright: { baseline: pair.baseline.owningPlaywright, candidate: pair.candidate.owningPlaywright } });
|
||||
}
|
||||
}
|
||||
const workloads = [
|
||||
{ name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
@@ -30,9 +109,128 @@ const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floo
|
||||
const first = pairs[0];
|
||||
const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
||||
const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value);
|
||||
function assertCaseAuthority(report: any, result: any) {
|
||||
const remote = report.remote, local = report.local;
|
||||
const authority = result.controllerAuthority;
|
||||
assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing controller authority proof before the measured epoch");
|
||||
assert.equal(authority?.kind, result.stop ? "host-file-journal" : "database-controller-checkpoint", "Wrong checkpoint authority for actual transport");
|
||||
assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction");
|
||||
assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation");
|
||||
assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0
|
||||
&& Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0
|
||||
&& authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs,
|
||||
"Close the actual remote authority snapshot before the measured epoch");
|
||||
assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof");
|
||||
assertParticipantTopology(report, result);
|
||||
assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch");
|
||||
assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority");
|
||||
const authorityByRun = new Map<string, any>();
|
||||
for (const row of authority.rows) {
|
||||
assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual task/lease UUID");
|
||||
assert.equal(row.company_id, report.companyId, "Remote authority company mismatch");
|
||||
assert.equal(row.environment_id, result.environmentId, "Remote authority environment mismatch");
|
||||
assert.equal(row.runtime_mode, "native");
|
||||
if (!result.stop) {
|
||||
assert.ok(remote.agentIds.includes(row.agent_id), "Remote run did not use its normal remote agent");
|
||||
assert.equal(row.provider, "fake-plugin");
|
||||
assert.match(row.provider_lease_id, /^fake-plugin:\/\//);
|
||||
assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header");
|
||||
assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0);
|
||||
} else {
|
||||
assert.equal(row.agent_id, local.controlAgentId, "Stop did not use its separate local control agent");
|
||||
assert.equal(row.provider, "local", "Stop requires its normal local lease");
|
||||
assert.equal(row.provider_lease_id, null); assert.equal(row.checkpoint_part, null, "Local Stop must retain host-file authority without a database header");
|
||||
assert.ok(BigInt(String(row.process_identity_seq)) > 0n);
|
||||
assert.equal(row.process_identity_source_event_id, null, "Local identity must be server-authored");
|
||||
assert.equal(row.process_identity?.targetKind, "local");
|
||||
assert.equal(row.process_identity?.processPid, row.process_pid);
|
||||
assert.equal(row.process_identity?.processGroupId, row.process_group_id);
|
||||
assert.equal(row.process_identity?.processStartedAt, row.process_started_at);
|
||||
assert.ok(uuid(row.admitted_execution_workspace_id), "Missing actual admitted local workspace binding");
|
||||
}
|
||||
authorityByRun.set(row.run_id, row);
|
||||
}
|
||||
return { authority, authorityByRun };
|
||||
}
|
||||
|
||||
function assertCaseSettlement(report: any, result: any, validated = assertCaseAuthority(report, result)) {
|
||||
const remote = report.remote, local = report.local;
|
||||
const { authority, authorityByRun } = validated;
|
||||
if (result.stop) {
|
||||
assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed");
|
||||
const row = authority.rows[0], journal = authority.localJournal, before = result.localProcesses;
|
||||
assert.equal(journal?.schema, "paperclip.runner.durable.control-plane-state.v1", "Missing owned local journal authority");
|
||||
assert.ok(sha256(journal.sha256) && journal.bytes > 0 && Number.isSafeInteger(journal.checkedAtMs)
|
||||
&& journal.checkedAtMs < result.epochMs, "Verify actual local file journal before the measured epoch");
|
||||
assert.notEqual(journal.checkpointEventPolicy, "control-v1", "Preserve the original local raw transport recovery policy");
|
||||
for (const [field, expected] of [["runId", row.run_id], ["runnerInstanceId", row.runner_instance_id],
|
||||
["normalizedSessionId", row.native_session_id], ["environmentLeaseId", row.admitted_execution_workspace_id]]) assert.equal(journal.identity?.[field], expected, "Local journal identity mismatch");
|
||||
assert.ok(before && before.hostname === row.process_identity.hostname && before.runnerPid === row.process_pid
|
||||
&& before.processGroupId === row.process_group_id && before.providerPid === result.providerClocks[0].providerPid,
|
||||
"Local observed process ownership mismatch");
|
||||
assert.ok(before.runnerAlive && before.groupAlive && before.providerAlive && Number.isSafeInteger(before.checkedAtMs)
|
||||
&& before.checkedAtMs < result.epochMs, "Independently observe all owned local processes before emission");
|
||||
const stopped = result.localStopProof, database = stopped?.database, after = database?.rows?.[0];
|
||||
assert.ok(stopped && Number.isSafeInteger(stopped.checkedAtMs) && stopped.checkedAtMs >= result.stopAcknowledgedAtMs
|
||||
&& stopped.checkedAtMs >= result.settledAtMs, "Verify local physical Stop after timing boundaries");
|
||||
assert.equal(database?.kind, "host-file-journal"); assert.equal(database?.observedReadOnly, "on");
|
||||
assert.equal(database?.observedIsolation, "repeatable read"); assert.equal(database?.rows?.length, 1);
|
||||
assert.equal(database?.checkedBeforeMeasuredEpoch, false);
|
||||
assert.ok(Number.isSafeInteger(database.checkedAtMs) && database.checkedAtMs >= result.stopAcknowledgedAtMs
|
||||
&& Number.isSafeInteger(database.connectionClosedAtMs) && database.connectionClosedAtMs >= database.checkedAtMs
|
||||
&& database.connectionClosedAtMs <= stopped.checkedAtMs, "Local stopped database proof predates acknowledgement");
|
||||
for (const field of ["run_id", "company_id", "environment_id", "lease_id", "agent_id", "provider", "provider_lease_id",
|
||||
"admitted_execution_workspace_id", "process_identity_seq", "process_identity_source_event_id"]) assert.equal(after?.[field], row[field], "Local stopped database identity mismatch");
|
||||
assert.equal(after.status, "cancelled"); assert.equal(after.cancellation_dispatch_state, "acknowledged");
|
||||
if (!after.process_pid && !after.process_group_id) {
|
||||
assert.ok(BigInt(String(after.process_stopped_seq)) > BigInt(String(row.process_identity_seq)), "Cleared local process identity needs its server stop receipt");
|
||||
assert.equal(after.process_stopped_source_event_id, null);
|
||||
assert.equal(after.process_stopped?.processPid, before.runnerPid);
|
||||
assert.equal(after.process_stopped?.processGroupId, before.processGroupId);
|
||||
assert.equal(after.process_stopped?.processStartedAt, row.process_identity.processStartedAt);
|
||||
} else {
|
||||
assert.equal(after.process_pid, before.runnerPid); assert.equal(after.process_group_id, before.processGroupId);
|
||||
}
|
||||
for (const field of ["runnerPid", "processGroupId", "providerPid", "hostname"]) assert.equal(stopped.processes?.[field], before[field]);
|
||||
assert.equal(stopped.processes.runnerAbsent, true); assert.equal(stopped.processes.groupAbsent, true);
|
||||
assert.equal(stopped.processes.providerAbsent, true, "Local provider remains alive after Stop");
|
||||
const lease = stopped.lease;
|
||||
assert.equal(lease?.id, row.lease_id); assert.equal(lease?.companyId, report.companyId);
|
||||
assert.equal(lease?.environmentId, local.environmentId); assert.equal(lease?.heartbeatRunId, row.run_id);
|
||||
assert.equal(lease?.provider, "local"); assert.equal(lease?.providerLeaseId, null);
|
||||
assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt
|
||||
&& lease.cleanupStatus !== "failed", "Local Stop lease cleanup remains unsettled");
|
||||
assert.equal(result.remoteLeases, undefined, "Local Stop cannot claim fake remote cleanup");
|
||||
} else {
|
||||
assert.equal(result.localStopProof, undefined, "Streaming cannot claim a local Stop proof");
|
||||
assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof");
|
||||
assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch");
|
||||
assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs,
|
||||
"Verify remote leases after recorded timing boundaries");
|
||||
for (const lease of result.remoteLeases) {
|
||||
const row = authorityByRun.get(lease.runId);
|
||||
assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId);
|
||||
assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider);
|
||||
assert.equal(lease.providerLeaseId, row.provider_lease_id);
|
||||
assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt
|
||||
&& lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled");
|
||||
if (lease.terminationReceipt) {
|
||||
const receipt = lease.terminationReceipt;
|
||||
assert.equal(receipt.schema, "paperclip.remote-termination.v1");
|
||||
assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId);
|
||||
assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider);
|
||||
assert.equal(receipt.providerLeaseId, lease.providerLeaseId);
|
||||
assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
|
||||
const report = pair[arm];
|
||||
assert.equal(report.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream", "Fresh browser acceptance requires the qualified v4 mixed clock protocol");
|
||||
assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : H4D_PROTOCOL,
|
||||
"Fresh browser acceptance requires the independent-agent observation or default-deadline protocol");
|
||||
assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport");
|
||||
assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, "Missing explicit transport and real cloud Stop qualification policy");
|
||||
@@ -83,6 +281,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.equal(report.setupActivationConcurrency, 2, "Keep the qualified setup activation concurrency of two");
|
||||
assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval");
|
||||
assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance");
|
||||
assertParticipantSetup(report);
|
||||
assert.equal(report.cleanup?.complete, true, "Acceptance requires completed fixture cleanup");
|
||||
assert.deepEqual(report.cleanup?.errors, [], "Acceptance rejects fixture cleanup errors");
|
||||
assert.notEqual(report.diagnosticOnly, true, "Diagnostic profiles cannot qualify unprofiled performance");
|
||||
@@ -97,7 +296,13 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm");
|
||||
assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples");
|
||||
assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples");
|
||||
if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples");
|
||||
assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs");
|
||||
assert.match(report.harnessProvenance?.sourceRevision ?? "", /^[a-f0-9]{40}$/, "Missing frozen harness source revision");
|
||||
assert.equal(report.harnessProvenance.sourceStatus, "");
|
||||
for (const field of ["sourceRevision", "sourceHashes", "driverLockSha256"])
|
||||
assert.deepEqual(report.harnessProvenance[field], first.baseline.harnessProvenance[field], "Harness source changed between arms");
|
||||
if (phase === "aa") assert.equal(report.sourceRevision, first.baseline.sourceRevision, "A/A must use one source revision");
|
||||
for (const name of names) {
|
||||
const result = report.cases.find((item: any) => item.name === name);
|
||||
@@ -124,44 +329,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
assert.ok(result.runIds.every(uuid), "Missing actual remote run UUID");
|
||||
assert.equal(new Set(result.runIds).size, result.runs, "Duplicate remote run identity");
|
||||
assert.deepEqual(result.durable.map((proof: any) => proof.runId).sort(), [...result.runIds].sort(), "Durable proof run set mismatch");
|
||||
const authority = result.controllerAuthority;
|
||||
assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing controller authority proof before the measured epoch");
|
||||
assert.equal(authority?.kind, result.stop ? "host-file-journal" : "database-controller-checkpoint", "Wrong checkpoint authority for actual transport");
|
||||
assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction");
|
||||
assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation");
|
||||
assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0
|
||||
&& Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0
|
||||
&& authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs,
|
||||
"Close the actual remote authority snapshot before the measured epoch");
|
||||
assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof");
|
||||
assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch");
|
||||
assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority");
|
||||
const authorityByRun = new Map<string, any>();
|
||||
for (const row of authority.rows) {
|
||||
assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual task/lease UUID");
|
||||
assert.equal(row.company_id, report.companyId, "Remote authority company mismatch");
|
||||
assert.equal(row.environment_id, result.environmentId, "Remote authority environment mismatch");
|
||||
assert.equal(row.runtime_mode, "native");
|
||||
if (!result.stop) {
|
||||
assert.ok(remote.agentIds.includes(row.agent_id), "Remote run did not use its normal remote agent");
|
||||
assert.equal(row.provider, "fake-plugin");
|
||||
assert.match(row.provider_lease_id, /^fake-plugin:\/\//);
|
||||
assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header");
|
||||
assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0);
|
||||
} else {
|
||||
assert.equal(row.agent_id, local.controlAgentId, "Stop did not use its separate local control agent");
|
||||
assert.equal(row.provider, "local", "Stop requires its normal local lease");
|
||||
assert.equal(row.provider_lease_id, null); assert.equal(row.checkpoint_part, null, "Local Stop must retain host-file authority without a database header");
|
||||
assert.ok(BigInt(String(row.process_identity_seq)) > 0n);
|
||||
assert.equal(row.process_identity_source_event_id, null, "Local identity must be server-authored");
|
||||
assert.equal(row.process_identity?.targetKind, "local");
|
||||
assert.equal(row.process_identity?.processPid, row.process_pid);
|
||||
assert.equal(row.process_identity?.processGroupId, row.process_group_id);
|
||||
assert.equal(row.process_identity?.processStartedAt, row.process_started_at);
|
||||
assert.ok(uuid(row.admitted_execution_workspace_id), "Missing actual admitted local workspace binding");
|
||||
}
|
||||
authorityByRun.set(row.run_id, row);
|
||||
}
|
||||
const { authority, authorityByRun } = assertCaseAuthority(report, result);
|
||||
assert.equal(result.providerClocks?.length, result.runs, "Every actual provider needs its wrapper launcher and PID");
|
||||
assert.deepEqual(result.providerClocks.map((clock: any) => clock.runId).sort(), [...result.runIds].sort());
|
||||
assert.equal(new Set(result.providerClocks.map((clock: any) => clock.providerPid)).size, result.runs);
|
||||
@@ -223,74 +391,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
}
|
||||
}
|
||||
assert.equal(result.settledAtMs, result.epochMs + result.drainThroughTerminalMs, "Terminal timing boundary changed");
|
||||
if (result.stop) {
|
||||
assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed");
|
||||
const row = authority.rows[0], journal = authority.localJournal, before = result.localProcesses;
|
||||
assert.equal(journal?.schema, "paperclip.runner.durable.control-plane-state.v1", "Missing owned local journal authority");
|
||||
assert.ok(sha256(journal.sha256) && journal.bytes > 0 && Number.isSafeInteger(journal.checkedAtMs)
|
||||
&& journal.checkedAtMs < result.epochMs, "Verify actual local file journal before the measured epoch");
|
||||
assert.notEqual(journal.checkpointEventPolicy, "control-v1", "Preserve the original local raw transport recovery policy");
|
||||
for (const [field, expected] of [["runId", row.run_id], ["runnerInstanceId", row.runner_instance_id],
|
||||
["normalizedSessionId", row.native_session_id], ["environmentLeaseId", row.admitted_execution_workspace_id]]) assert.equal(journal.identity?.[field], expected, "Local journal identity mismatch");
|
||||
assert.ok(before && before.hostname === row.process_identity.hostname && before.runnerPid === row.process_pid
|
||||
&& before.processGroupId === row.process_group_id && before.providerPid === result.providerClocks[0].providerPid,
|
||||
"Local observed process ownership mismatch");
|
||||
assert.ok(before.runnerAlive && before.groupAlive && before.providerAlive && Number.isSafeInteger(before.checkedAtMs)
|
||||
&& before.checkedAtMs < result.epochMs, "Independently observe all owned local processes before emission");
|
||||
const stopped = result.localStopProof, database = stopped?.database, after = database?.rows?.[0];
|
||||
assert.ok(stopped && Number.isSafeInteger(stopped.checkedAtMs) && stopped.checkedAtMs >= result.stopAcknowledgedAtMs
|
||||
&& stopped.checkedAtMs >= result.settledAtMs, "Verify local physical Stop after timing boundaries");
|
||||
assert.equal(database?.kind, "host-file-journal"); assert.equal(database?.observedReadOnly, "on");
|
||||
assert.equal(database?.observedIsolation, "repeatable read"); assert.equal(database?.rows?.length, 1);
|
||||
assert.equal(database?.checkedBeforeMeasuredEpoch, false);
|
||||
assert.ok(Number.isSafeInteger(database.checkedAtMs) && database.checkedAtMs >= result.stopAcknowledgedAtMs
|
||||
&& Number.isSafeInteger(database.connectionClosedAtMs) && database.connectionClosedAtMs >= database.checkedAtMs
|
||||
&& database.connectionClosedAtMs <= stopped.checkedAtMs, "Local stopped database proof predates acknowledgement");
|
||||
for (const field of ["run_id", "company_id", "environment_id", "lease_id", "agent_id", "provider", "provider_lease_id",
|
||||
"admitted_execution_workspace_id", "process_identity_seq", "process_identity_source_event_id"]) assert.equal(after?.[field], row[field], "Local stopped database identity mismatch");
|
||||
assert.equal(after.status, "cancelled"); assert.equal(after.cancellation_dispatch_state, "acknowledged");
|
||||
if (!after.process_pid && !after.process_group_id) {
|
||||
assert.ok(BigInt(String(after.process_stopped_seq)) > BigInt(String(row.process_identity_seq)), "Cleared local process identity needs its server stop receipt");
|
||||
assert.equal(after.process_stopped_source_event_id, null);
|
||||
assert.equal(after.process_stopped?.processPid, before.runnerPid);
|
||||
assert.equal(after.process_stopped?.processGroupId, before.processGroupId);
|
||||
assert.equal(after.process_stopped?.processStartedAt, row.process_identity.processStartedAt);
|
||||
} else {
|
||||
assert.equal(after.process_pid, before.runnerPid); assert.equal(after.process_group_id, before.processGroupId);
|
||||
}
|
||||
for (const field of ["runnerPid", "processGroupId", "providerPid", "hostname"]) assert.equal(stopped.processes?.[field], before[field]);
|
||||
assert.equal(stopped.processes.runnerAbsent, true); assert.equal(stopped.processes.groupAbsent, true);
|
||||
assert.equal(stopped.processes.providerAbsent, true, "Local provider remains alive after Stop");
|
||||
const lease = stopped.lease;
|
||||
assert.equal(lease?.id, row.lease_id); assert.equal(lease?.companyId, report.companyId);
|
||||
assert.equal(lease?.environmentId, local.environmentId); assert.equal(lease?.heartbeatRunId, row.run_id);
|
||||
assert.equal(lease?.provider, "local"); assert.equal(lease?.providerLeaseId, null);
|
||||
assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt
|
||||
&& lease.cleanupStatus !== "failed", "Local Stop lease cleanup remains unsettled");
|
||||
assert.equal(result.remoteLeases, undefined, "Local Stop cannot claim fake remote cleanup");
|
||||
} else {
|
||||
assert.equal(result.localStopProof, undefined, "Streaming cannot claim a local Stop proof");
|
||||
assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof");
|
||||
assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch");
|
||||
assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs,
|
||||
"Verify remote leases after recorded timing boundaries");
|
||||
for (const lease of result.remoteLeases) {
|
||||
const row = authorityByRun.get(lease.runId);
|
||||
assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId);
|
||||
assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider);
|
||||
assert.equal(lease.providerLeaseId, row.provider_lease_id);
|
||||
assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt
|
||||
&& lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled");
|
||||
if (lease.terminationReceipt) {
|
||||
const receipt = lease.terminationReceipt;
|
||||
assert.equal(receipt.schema, "paperclip.remote-termination.v1");
|
||||
assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId);
|
||||
assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider);
|
||||
assert.equal(receipt.providerLeaseId, lease.providerLeaseId);
|
||||
assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt");
|
||||
}
|
||||
}
|
||||
}
|
||||
assertCaseSettlement(report, result, { authority, authorityByRun });
|
||||
for (const proof of result.durable) {
|
||||
assert.equal(proof.ordered, true); assert.equal(proof.uniqueSourceIdentities, proof.individualDeltas);
|
||||
assert.equal(proof.matchedSourceClocks, proof.individualDeltas, "Every durable measured delta needs an original provider clock");
|
||||
@@ -300,6 +401,61 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`);
|
||||
}
|
||||
}
|
||||
let originalH4dConfirmation: Record<string, any> | undefined;
|
||||
if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
originalH4dConfirmation = { required: true, passed: false, reason: "Later H4d default-mode full functional proof on the same source and geometry is pending" };
|
||||
if (arg("original-h4d-proof")) {
|
||||
try {
|
||||
assert.ok(arg("original-h4d-inventory"), "H4d original-deadline proof requires its full artifact inventory sidecar");
|
||||
const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4d-proof")!));
|
||||
assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.equal(proof.protocolVersion, H4D_PROTOCOL, "Old H4c geometry cannot confirm H5b");
|
||||
assertParticipantSetup(proof);
|
||||
assert.deepEqual(proof.agentTopology, first.candidate.agentTopology);
|
||||
assert.equal(proof.observationMode, undefined, "Confirmation must use H4d default deadlines");
|
||||
assert.equal(proof.harnessProvenance?.sourceRevision, first.candidate.harnessProvenance.sourceRevision, "H4d confirmation must use the same frozen harness source");
|
||||
assert.deepEqual(proof.harnessProvenance.sourceHashes, first.candidate.harnessProvenance.sourceHashes);
|
||||
assert.equal(proof.harnessProvenance.driverLockSha256, first.candidate.harnessProvenance.driverLockSha256);
|
||||
assert.equal(proof.harnessProvenance.sourceStatus, "");
|
||||
assert.equal(proof.measurementSha256, first.candidate.measurementSha256, "H4d confirmation must bind the same measured harness bytes");
|
||||
assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false);
|
||||
assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled");
|
||||
assert.equal(proof.cleanup?.complete, true); assert.deepEqual(proof.cleanup.errors, []);
|
||||
assert.deepEqual(proof.browserRecording, first.candidate.browserRecording);
|
||||
assert.equal(proof.sourceRevision, first.candidate.sourceRevision); assert.equal(proof.compiledBuildRevision, first.candidate.compiledBuildRevision);
|
||||
assert.equal(proof.serverEntrySha256, first.candidate.serverEntrySha256);
|
||||
assert.deepEqual(proof.compiledHotPathHashes, first.candidate.compiledHotPathHashes);
|
||||
assert.equal(proof.transport, first.candidate.transport); assert.deepEqual(proof.transportPolicy, first.candidate.transportPolicy);
|
||||
assert.deepEqual(proof.cases.map((item: any) => item.name), names, "Original confirmation must include all six workloads");
|
||||
for (const fixed of workloads) {
|
||||
const item = proof.cases.find((item: any) => item.name === fixed.name);
|
||||
assertCaseSettlement(proof, item);
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(item[field], fixed[field]);
|
||||
assert.equal(item.persistedAfterReload, true); assert.equal(item.durable.length, fixed.runs);
|
||||
assert.equal(item.matchedVisibleClocks, item.visible.length);
|
||||
assert.ok(item.visible.length >= (fixed.stop ? 3 : fixed.events));
|
||||
assert.equal(item.runIds.length, fixed.runs); assert.equal(new Set(item.runIds).size, fixed.runs);
|
||||
for (const row of item.durable) {
|
||||
assert.equal(row.ordered, true); assert.equal(row.uniqueSourceIdentities, row.individualDeltas);
|
||||
assert.equal(row.matchedSourceClocks, row.individualDeltas);
|
||||
if (!fixed.stop) assert.equal(row.individualDeltas, fixed.events);
|
||||
}
|
||||
}
|
||||
const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs));
|
||||
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
|
||||
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4d-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
|
||||
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
|
||||
originalH4dConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision };
|
||||
} catch (error) {
|
||||
originalH4dConfirmation.reason = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
const metric = (item: any, key: string): number => key.split(".").reduce((value, part) => value[part], item);
|
||||
const measurements = names.flatMap(name => {
|
||||
const observed = first.baseline.cases.find((item: any) => item.name === name);
|
||||
@@ -323,7 +479,8 @@ const measurements = names.flatMap(name => {
|
||||
});
|
||||
});
|
||||
const result: Record<string, any> = {
|
||||
scope: "Compiled-app remote streaming and original local Stop performance; additional real cloud Stop qualification is required separately",
|
||||
scope: "Compiled-app streaming with100 independent normal agents and the original local Stop workload; real cloud Stop is separately required",
|
||||
agentTopology: first.baseline.agentTopology,
|
||||
transportPolicy: first.baseline.transportPolicy, additionalRealCloudStopQualificationRequired: true,
|
||||
admissionProtocol: first.baseline.admissionProtocol, setupActivationConcurrency: first.baseline.setupActivationConcurrency,
|
||||
protocolVersion: first.baseline.protocolVersion, transport: first.baseline.transport,
|
||||
@@ -334,11 +491,21 @@ const result: Record<string, any> = {
|
||||
comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
|
||||
manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements,
|
||||
primaryLoadedWorkload: "burst512", complete: true,
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments,
|
||||
artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4dConfirmation,
|
||||
originalProtocolAcceptance: "Baseline SLA misses are retained; H5b complete observation is not an original-deadline PASS",
|
||||
finalH4dFunctionalPassRequired: candidatePolicy === "final-b2" } : {}),
|
||||
};
|
||||
if (phase === "compare") {
|
||||
assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report");
|
||||
const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8"));
|
||||
assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true);
|
||||
if (h5) {
|
||||
assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256);
|
||||
assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required");
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"])
|
||||
assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A");
|
||||
}
|
||||
assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol");
|
||||
assert.equal(noise.transport, result.transport, "Noise report uses a different transport");
|
||||
assert.deepEqual(noise.transportPolicy, result.transportPolicy, "Transport policy changed after A/A calibration");
|
||||
@@ -362,10 +529,17 @@ if (phase === "compare") {
|
||||
noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
|
||||
passed: improvementRequired ? improvement : noRegression };
|
||||
});
|
||||
result.acceptancePassed = result.gates.every((entry: any) => entry.passed);
|
||||
result.performancePassed = result.gates.every((entry: any) => entry.passed)
|
||||
&& (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed));
|
||||
result.acceptancePassed = result.performancePassed && (!originalH4dConfirmation || originalH4dConfirmation.passed);
|
||||
}
|
||||
if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (process.argv.includes("--require-performance")) {
|
||||
assert.equal(h5, true, "H5b performance-only gating is explicit; default H4d keeps original deadlines");
|
||||
assert.equal(phase, "compare");
|
||||
assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose");
|
||||
}
|
||||
if (process.argv.includes("--require-acceptance")) {
|
||||
assert.equal(phase, "compare", "A/A calibrates noise; it is not performance acceptance");
|
||||
assert.equal(result.acceptancePassed, true, "Browser performance acceptance failed; retain results and diagnose");
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { H5_REGISTRATION, H5_WORKLOAD_SHA256 } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
if (process.argv.length > 2) throw new Error('Dry run only: this command accepts no execution arguments');
|
||||
console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadSha256: H5_WORKLOAD_SHA256,
|
||||
registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes',
|
||||
'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation',
|
||||
'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass',
|
||||
'Later H4d default-mode full functional pass on the same source, geometry and final SUT'] }, null, 2));
|
||||
@@ -0,0 +1,137 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
const SCHEMA = 'paperclip.native-ingestion.arm-inventory.v1';
|
||||
const ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const sha256 = value => createHash('sha256').update(value).digest('hex');
|
||||
const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
||||
const revision = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value);
|
||||
const object = value => value && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
async function bounded(file, maximumBytes) {
|
||||
const metadata = await stat(file);
|
||||
assert.ok(metadata.isFile() && metadata.size <= maximumBytes, 'Inventory evidence file is absent, nonregular or oversized');
|
||||
const bytes = await readFile(file);
|
||||
assert.ok(bytes.length <= maximumBytes, 'Inventory evidence file grew beyond its bound');
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function relativeArtifact(value) {
|
||||
return typeof value === 'string' && value.length > 0 && !value.includes('\\')
|
||||
&& !value.includes('\0') && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value
|
||||
&& !value.split('/').includes('..') && ROOTS.some(root => value.startsWith(`${root}/`));
|
||||
}
|
||||
|
||||
// The existing Python generator sorts Unicode code points and uses json.dumps
|
||||
// with sort_keys=True, separators=(',', ':') and its default ensure_ascii=True.
|
||||
function pythonOrder(left, right) {
|
||||
const a = [...left], b = [...right];
|
||||
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
||||
const difference = a[index].codePointAt(0) - b[index].codePointAt(0);
|
||||
if (difference) return difference;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })
|
||||
.replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
||||
|
||||
/**
|
||||
* Validates frozen inventory evidence and reported serving fingerprints only.
|
||||
* The owner supplies this sidecar per arm; this helper never reads the SUT tree
|
||||
* and does not attest that a live filesystem remained unchanged during a run.
|
||||
*/
|
||||
export async function validateArmInventory(sidecarFile, { evidence, owningPlaywright }) {
|
||||
assert.equal(typeof sidecarFile, 'string', 'Missing arm inventory sidecar');
|
||||
assert.ok(object(evidence) && object(owningPlaywright), 'Missing serving evidence or owning Playwright report');
|
||||
const absoluteSidecar = path.resolve(sidecarFile), base = path.dirname(absoluteSidecar);
|
||||
const sidecarBytes = await bounded(absoluteSidecar, 256 * 1024);
|
||||
const sidecar = JSON.parse(sidecarBytes.toString('utf8'));
|
||||
assert.equal(sidecar.schema, SCHEMA, 'Unsupported arm inventory sidecar schema');
|
||||
assert.ok(['B0', 'B1', 'B2'].includes(sidecar.variant), 'Unknown inventory variant');
|
||||
assert.ok(revision(sidecar.sourceRevision) && revision(sidecar.compiledBuildRevision), 'Invalid inventory source revision');
|
||||
assert.equal(sidecar.sourceRevision, evidence.sourceRevision, 'Inventory source revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, evidence.compiledBuildRevision, 'Inventory build revision differs from arm');
|
||||
assert.equal(sidecar.compiledBuildRevision, sidecar.sourceRevision, 'Inventory was built from a different source revision');
|
||||
assert.ok(digest(sidecar.playwrightReportSha256) && digest(owningPlaywright.reportSha256), 'Missing owning Playwright report hash');
|
||||
assert.equal(sidecar.playwrightReportSha256, owningPlaywright.reportSha256, 'Inventory belongs to another owning Playwright report');
|
||||
const resolve = name => {
|
||||
assert.ok(typeof sidecar[name] === 'string' && sidecar[name].length > 0, `Missing inventory ${name}`);
|
||||
return path.resolve(base, sidecar[name]);
|
||||
};
|
||||
const readHashed = async (name, hashName, maximumBytes) => {
|
||||
assert.ok(digest(sidecar[hashName]), `Invalid inventory ${hashName}`);
|
||||
const file = resolve(name), bytes = await bounded(file, maximumBytes);
|
||||
assert.equal(sha256(bytes), sidecar[hashName], `Inventory ${name} bytes changed`);
|
||||
return { file, bytes };
|
||||
};
|
||||
const manifestSource = await readHashed('manifestFile', 'manifestFileSha256', 64 * 1024 * 1024);
|
||||
const manifest = JSON.parse(gunzipSync(manifestSource.bytes, { maxOutputLength: 128 * 1024 * 1024 }).toString('utf8'));
|
||||
assert.deepEqual(Object.keys(manifest).sort(), ['aggregateSha256', 'compiledBuildRevision', 'fileCount', 'files', 'revision', 'roots', 'variant']);
|
||||
assert.equal(manifest.variant, sidecar.variant, 'Inventory manifest variant differs from sidecar');
|
||||
assert.equal(manifest.revision, sidecar.sourceRevision, 'Inventory manifest source revision differs from arm');
|
||||
assert.equal(manifest.compiledBuildRevision, sidecar.compiledBuildRevision, 'Inventory manifest build revision differs from arm');
|
||||
assert.deepEqual(manifest.roots, ROOTS, 'Full registered inventory roots are required');
|
||||
assert.ok(Number.isSafeInteger(manifest.fileCount) && manifest.fileCount > 0 && manifest.fileCount <= 500000, 'Invalid inventory file count');
|
||||
assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount, 'Inventory file count is incomplete');
|
||||
assert.ok(digest(manifest.aggregateSha256) && digest(sidecar.aggregateSha256), 'Invalid inventory aggregate hash');
|
||||
const files = new Map();
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(object(row), 'Invalid inventory file row');
|
||||
assert.deepEqual(Object.keys(row).sort(), ['bytes', 'path', 'sha256']);
|
||||
assert.ok(relativeArtifact(row.path), 'Inventory file escapes registered roots');
|
||||
assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0 && digest(row.sha256), 'Invalid inventory file descriptor');
|
||||
assert.equal(files.has(row.path), false, 'Duplicate inventory file path');
|
||||
files.set(row.path, row);
|
||||
}
|
||||
const sorted = [...manifest.files].sort((left, right) => pythonOrder(left.path, right.path));
|
||||
assert.deepEqual(manifest.files.map(row => row.path), sorted.map(row => row.path), 'Inventory file order differs from generator');
|
||||
for (const root of ROOTS) assert.ok(manifest.files.some(row => row.path.startsWith(`${root}/`)), 'Inventory omitted a registered root');
|
||||
const aggregate = sha256(Buffer.from(`${sorted.map(pythonRow).join('\n')}\n`, 'utf8'));
|
||||
assert.equal(aggregate, manifest.aggregateSha256, 'Inventory aggregate does not match exact file descriptors');
|
||||
assert.equal(aggregate, sidecar.aggregateSha256, 'Inventory aggregate differs from sidecar');
|
||||
assert.ok(digest(evidence.serverEntrySha256), 'Missing serving server entry hash');
|
||||
assert.equal(files.get('server/dist/index.js')?.sha256, evidence.serverEntrySha256, 'Serving server entry differs from full inventory');
|
||||
assert.ok(object(evidence.compiledHotPathHashes) && Object.keys(evidence.compiledHotPathHashes).length > 0, 'Missing serving hot-path fingerprints');
|
||||
for (const [file, hash] of Object.entries(evidence.compiledHotPathHashes)) {
|
||||
assert.ok(relativeArtifact(file) && digest(hash), 'Invalid serving hot-path fingerprint');
|
||||
assert.equal(files.get(file)?.sha256, hash, 'Serving hot path differs from full inventory');
|
||||
}
|
||||
const summarySource = await readHashed('inventorySummaryFile', 'inventorySummarySha256', 1024 * 1024);
|
||||
const summary = JSON.parse(summarySource.bytes.toString('utf8'));
|
||||
const summaryRow = summary.inventories?.[sidecar.variant];
|
||||
assert.ok(object(summaryRow), 'Inventory summary omitted this variant');
|
||||
assert.ok(typeof summaryRow.manifest === 'string' && path.basename(summaryRow.manifest) === summaryRow.manifest
|
||||
&& summaryRow.manifest.endsWith('.json.gz'), 'Inventory summary manifest reference is invalid');
|
||||
assert.equal(summaryRow.manifestSha256, sidecar.manifestFileSha256, 'Inventory summary points to different manifest bytes');
|
||||
assert.equal(summaryRow.aggregateSha256, aggregate, 'Inventory summary aggregate differs');
|
||||
assert.equal(summaryRow.fileCount, manifest.fileCount, 'Inventory summary file count differs');
|
||||
const receiptSource = await readHashed('qualifiedReceiptFile', 'qualifiedReceiptSha256', 4 * 1024 * 1024);
|
||||
const receipt = JSON.parse(receiptSource.bytes.toString('utf8'));
|
||||
assert.ok(Array.isArray(receipt.results), 'Qualified build receipt has no results');
|
||||
const qualified = receipt.results.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(qualified.length, 1, 'Qualified build receipt must identify this variant exactly once');
|
||||
assert.equal(qualified[0].revision, sidecar.sourceRevision, 'Qualified build receipt source revision differs');
|
||||
assert.equal(qualified[0].compiledBuildRevision, sidecar.compiledBuildRevision, 'Qualified build receipt build revision differs');
|
||||
assert.equal(qualified[0].sourceClean, true, 'Qualified build source was not clean');
|
||||
assert.ok(typeof receipt.artifactInventorySummary === 'string' && receipt.artifactInventorySummary.length > 0,
|
||||
'Qualified build receipt omitted its inventory summary reference');
|
||||
assert.ok(revision(receipt.rustSourceTree), 'Qualified build receipt omitted Rust input identity');
|
||||
assert.ok(Array.isArray(receipt.nativeStageHashes), 'Qualified build receipt omitted native stage hashes');
|
||||
const native = receipt.nativeStageHashes.filter(row => row.variant === sidecar.variant);
|
||||
assert.equal(native.length, 1, 'Qualified build receipt must identify this native stage exactly once');
|
||||
assert.ok(digest(native[0].binarySha256) && digest(receipt.binarySha256), 'Qualified native binary hash is invalid');
|
||||
assert.equal(native[0].binarySha256, receipt.binarySha256, 'Qualified native binary hashes disagree');
|
||||
assert.equal(native[0].binarySha256, evidence.remote?.runner?.sha256, 'Serving native binary differs from qualified build');
|
||||
return { schema: SCHEMA, sidecarFile: absoluteSidecar, sidecarFileSha256: sha256(sidecarBytes), variant: sidecar.variant,
|
||||
revision: manifest.revision, compiledBuildRevision: manifest.compiledBuildRevision,
|
||||
manifestFile: manifestSource.file, manifestFileSha256: sidecar.manifestFileSha256,
|
||||
aggregateSha256: aggregate, fileCount: manifest.fileCount, roots: manifest.roots,
|
||||
inventorySummaryFile: summarySource.file, inventorySummarySha256: sidecar.inventorySummarySha256,
|
||||
qualifiedReceiptFile: receiptSource.file, qualifiedReceiptSha256: sidecar.qualifiedReceiptSha256,
|
||||
binarySha256: native[0].binarySha256, rustSourceTree: receipt.rustSourceTree,
|
||||
playwrightReportSha256: owningPlaywright.reportSha256,
|
||||
scope: 'Exact frozen manifest/summary/build-receipt bytes for all registered roots, with reported server/hot-path/native fingerprints checked; no live SUT filesystem attestation' };
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import { validateArmInventory } from './native-ingestion-h5-inventory.mjs';
|
||||
|
||||
const hash = value => createHash('sha256').update(value).digest('hex');
|
||||
const sourceRevision = '1'.repeat(40), reportSha256 = hash('owning-playwright'), binarySha256 = hash('runner');
|
||||
const roots = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
const descriptors = () => [
|
||||
'packages/db/src/client.ts', 'packages/shared/src/index.ts', 'server/dist/hot-a.js',
|
||||
'server/dist/hot-b.js', 'server/dist/hot-c.js', 'server/dist/index.js', 'ui/dist/index.html',
|
||||
].map(file => ({ bytes: 10, path: file, sha256: hash(file) }));
|
||||
const aggregate = files => hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join('\n') + '\n');
|
||||
|
||||
async function fixture(directory, options = {}) {
|
||||
const revision = options.revision ?? sourceRevision;
|
||||
const files = descriptors(), manifest = { variant: 'B2', revision, compiledBuildRevision: revision,
|
||||
roots, fileCount: files.length, aggregateSha256: aggregate(files), files };
|
||||
options.manifest?.(manifest);
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify(manifest)));
|
||||
const summary = { capturedAt: '2026-11-03T00:00:00.000Z', inventories: { B2: {
|
||||
manifest: 'B2-artifact-inventory.json.gz', manifestSha256: hash(manifestBytes), fileCount: manifest.fileCount,
|
||||
aggregateSha256: manifest.aggregateSha256 } } };
|
||||
options.summary?.(summary);
|
||||
const receipt = { completedAt: '2026-11-03T00:00:01.000Z',
|
||||
results: [{ variant: 'B2', revision, compiledBuildRevision: revision, sourceClean: true }],
|
||||
rustSourceTree: '2'.repeat(40), binarySha256, nativeStageHashes: [{ variant: 'B2', binarySha256 }],
|
||||
artifactInventorySummary: '/owned/original/artifact-inventory-summary.json' };
|
||||
options.receipt?.(receipt);
|
||||
const summaryBytes = Buffer.from(JSON.stringify(summary)), receiptBytes = Buffer.from(JSON.stringify(receipt));
|
||||
const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant: 'B2', sourceRevision: revision,
|
||||
compiledBuildRevision: revision, playwrightReportSha256: reportSha256,
|
||||
manifestFile: 'B2-artifact-inventory.json.gz', manifestFileSha256: hash(manifestBytes),
|
||||
aggregateSha256: manifest.aggregateSha256,
|
||||
qualifiedReceiptFile: 'qualified.json', qualifiedReceiptSha256: hash(receiptBytes),
|
||||
inventorySummaryFile: 'artifact-inventory-summary.json', inventorySummarySha256: hash(summaryBytes) };
|
||||
options.sidecar?.(sidecar);
|
||||
const evidence = { sourceRevision: revision, compiledBuildRevision: revision,
|
||||
serverEntrySha256: hash('server/dist/index.js'), remote: { runner: { sha256: binarySha256 } },
|
||||
compiledHotPathHashes: Object.fromEntries(['server/dist/hot-a.js', 'server/dist/hot-b.js', 'server/dist/hot-c.js'].map(file => [file, hash(file)])) };
|
||||
options.evidence?.(evidence);
|
||||
await Promise.all([
|
||||
writeFile(path.join(directory, sidecar.manifestFile), manifestBytes),
|
||||
writeFile(path.join(directory, sidecar.inventorySummaryFile), summaryBytes),
|
||||
writeFile(path.join(directory, sidecar.qualifiedReceiptFile), receiptBytes),
|
||||
writeFile(path.join(directory, 'arm-inventory.json'), JSON.stringify(sidecar)),
|
||||
]);
|
||||
return { sidecarFile: path.join(directory, 'arm-inventory.json'), sidecar, evidence,
|
||||
owningPlaywright: { reportSha256 }, manifest, summary, receipt };
|
||||
}
|
||||
|
||||
async function withFixture(options, work) {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'h5-inventory-'));
|
||||
try { await work(await fixture(directory, options), directory); }
|
||||
finally { await rm(directory, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
test('full frozen inventory, qualified native build and owning report bind one arm', async () => {
|
||||
await withFixture({}, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.manifestFileSha256, value.sidecar.manifestFileSha256);
|
||||
assert.equal(proof.aggregateSha256, aggregate(descriptors()));
|
||||
assert.equal(proof.fileCount, descriptors().length);
|
||||
assert.equal(proof.qualifiedReceiptSha256, value.sidecar.qualifiedReceiptSha256);
|
||||
assert.equal(proof.playwrightReportSha256, reportSha256);
|
||||
assert.equal(proof.binarySha256, binarySha256);
|
||||
assert.deepEqual(proof.roots, roots);
|
||||
});
|
||||
});
|
||||
|
||||
test('exact compressed manifest bytes and build-receipt bytes cannot be swapped', async () => {
|
||||
for (const field of ['manifestFile', 'qualifiedReceiptFile']) await withFixture({}, async (value, directory) => {
|
||||
const file = path.join(directory, value.sidecar[field]);
|
||||
await writeFile(file, Buffer.concat([await readFile(file), Buffer.from(' ')]));
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /bytes changed/);
|
||||
});
|
||||
});
|
||||
|
||||
test('source, compiled build and owning report must match the arm', async () => {
|
||||
const cases = [
|
||||
[{ sidecar: row => { row.sourceRevision = '3'.repeat(40); } }, /source revision differs/],
|
||||
[{ sidecar: row => { row.compiledBuildRevision = '3'.repeat(40); } }, /build revision differs/],
|
||||
[{ sidecar: row => { row.playwrightReportSha256 = hash('another-report'); } }, /another owning Playwright/],
|
||||
[{ manifest: row => { row.revision = '3'.repeat(40); } }, /manifest source revision/],
|
||||
[{ receipt: row => { row.results[0].revision = '3'.repeat(40); } }, /receipt source revision/],
|
||||
[{ receipt: row => { row.results[0].compiledBuildRevision = '3'.repeat(40); } }, /receipt build revision/],
|
||||
[{ receipt: row => { row.results[0].sourceClean = false; } }, /source was not clean/],
|
||||
];
|
||||
for (const [options, message] of cases) await withFixture(options, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('file descriptors must be complete, bounded, distinct and match the Python aggregate', async () => {
|
||||
const cases = [
|
||||
[row => { row.fileCount--; }, /file count is incomplete/],
|
||||
[row => { row.aggregateSha256 = hash('wrong aggregate'); }, /aggregate does not match/],
|
||||
[row => { row.files[0].bytes = -1; }, /file descriptor/],
|
||||
[row => { row.files[0].path = '../outside'; }, /escapes registered roots/],
|
||||
[row => { row.files[1].path = row.files[0].path; }, /Duplicate/],
|
||||
[row => { row.files.reverse(); }, /order differs/],
|
||||
[row => { row.files.pop(); row.fileCount--; row.aggregateSha256 = aggregate(row.files); }, /omitted a registered root/],
|
||||
[row => { row.roots = ['server/dist']; }, /registered inventory roots/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ manifest: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
});
|
||||
|
||||
test('summary identifies exact gzip, count and aggregate for the selected variant', async () => {
|
||||
for (const [key, value] of [['manifestSha256', hash('wrong')], ['fileCount', 1], ['aggregateSha256', hash('wrong')]]) {
|
||||
await withFixture({ summary: row => { row.inventories.B2[key] = value; } }, async fixture => {
|
||||
await assert.rejects(validateArmInventory(fixture.sidecarFile, fixture), /Inventory summary/);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('serving entry, every hot path and native binary must match full evidence', async () => {
|
||||
const cases = [
|
||||
[row => { row.serverEntrySha256 = hash('wrong-entry'); }, /server entry differs/],
|
||||
[row => { row.compiledHotPathHashes['server/dist/hot-c.js'] = hash('wrong-hot-path'); }, /hot path differs/],
|
||||
[row => { row.remote.runner.sha256 = hash('wrong-native'); }, /native binary differs/],
|
||||
];
|
||||
for (const [change, message] of cases) await withFixture({ evidence: change }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), message);
|
||||
});
|
||||
await withFixture({ receipt: row => { row.nativeStageHashes.push({ ...row.nativeStageHashes[0] }); } }, async value => {
|
||||
await assert.rejects(validateArmInventory(value.sidecarFile, value), /native stage exactly once/);
|
||||
});
|
||||
});
|
||||
|
||||
test('a future candidate revision is accepted only with matching complete provenance', async () => {
|
||||
await withFixture({ revision: 'f'.repeat(40) }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.revision, 'f'.repeat(40));
|
||||
assert.equal(proof.compiledBuildRevision, 'f'.repeat(40));
|
||||
});
|
||||
});
|
||||
|
||||
test('aggregate follows the existing Python ensure_ascii convention for non-ASCII paths', async () => {
|
||||
// Independently computed with the retained artifact-inventory-ce33.py
|
||||
// json.dumps(sort_keys=True, separators=(',', ':')) canonical convention.
|
||||
const expected = '6f7bf8cab07e2aaf8faaa95a8bdb565c588670e4e3e3580acdba1f93154fedde';
|
||||
await withFixture({ manifest: row => {
|
||||
row.files.at(-1).path = 'ui/dist/é😀\u007f.html'; row.aggregateSha256 = expected;
|
||||
} }, async value => {
|
||||
const proof = await validateArmInventory(value.sidecarFile, value);
|
||||
assert.equal(proof.aggregateSha256, expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { createRequire } from 'node:module';
|
||||
import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate,
|
||||
H5_LIMITS, ORIGINAL_BUDGETS, prepareParticipantAgents } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
|
||||
const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`),
|
||||
{ matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } });
|
||||
|
||||
test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
assert.equal(require('@playwright/test/package.json').version, '1.62.1');
|
||||
await assert.rejects(expect.poll(async () => 0, { timeout: 15, intervals: [5] }).toBeGreaterThanOrEqual(1), error => {
|
||||
assert.equal(error.matcherResult?.name, 'toBeGreaterThanOrEqual');
|
||||
assert.equal(error.matcherResult.pass, false);
|
||||
assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate'));
|
||||
return true;
|
||||
});
|
||||
// This checks the real library's format; it does not execute a120s wait or
|
||||
// claim runtime qualification of H5's extended application path.
|
||||
});
|
||||
|
||||
test('real frozen Playwright can report its configured timeout before the nominal deadline', async context => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
assert.equal(require('@playwright/test/package.json').version, '1.62.1');
|
||||
const timeoutMs = 1200, intervalMs = 1500, startedAtMs = Date.now(), started = performance.now();
|
||||
let observations = 0, failure;
|
||||
try {
|
||||
await expect.poll(async () => { observations++; return 0; }, { timeout: timeoutMs, intervals: [intervalMs] }).toBeGreaterThanOrEqual(1);
|
||||
} catch (error) { failure = error; }
|
||||
const assessedAtMs = Date.now(), elapsedMs = performance.now() - started;
|
||||
assert.equal(failure?.matcherResult?.name, 'toBeGreaterThanOrEqual');
|
||||
assert.equal(failure.matcherResult.pass, false);
|
||||
assert.ok(failure.message.includes(`Timeout ${timeoutMs}ms exceeded while waiting on the predicate`));
|
||||
assert.equal(observations, 1);
|
||||
assert.ok(assessedAtMs < startedAtMs + timeoutMs, 'The real poll ends before its nominal deadline when its next interval would cross it');
|
||||
context.diagnostic(JSON.stringify({ scope: 'Small real-library poll only; no app or120s runtime qualification',
|
||||
playwrightVersion: '1.62.1', timeoutMs, intervalMs, startedAtMs, nominalDeadlineMs: startedAtMs + timeoutMs,
|
||||
assessedAtMs, elapsedMs, observations, matcherName: failure.matcherResult.name, matcherPassed: failure.matcherResult.pass }));
|
||||
});
|
||||
|
||||
test('H5b requires its explicit mode and leaves default H4d selected', () => {
|
||||
assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true);
|
||||
assert.throws(() => h5Enabled('almost-h5'));
|
||||
assert.throws(() => h5Enabled('h5-observe-original-sla'), 'Old H5a observations use a different geometry');
|
||||
});
|
||||
|
||||
for (const fail of [false, true]) test(`participant setup uses at most8 workers, preserves index order and settles all attempts${fail ? ' after a failure' : ''}`, async () => {
|
||||
const count = 20, pending = [], records = []; let active = 0, maximum = 0, finished = 0;
|
||||
const completion = prepareParticipantAgents({ count, onOutcome: value => records.push(structuredClone(value)),
|
||||
createAndReady: async index => {
|
||||
active++; maximum = Math.max(maximum, active);
|
||||
await new Promise(resolve => pending.push(resolve)); active--; finished++;
|
||||
if (fail && index === 3) throw new Error('actual agent readiness failed');
|
||||
return { id: `agent-${index}` };
|
||||
} }).then(value => ({ value }), error => ({ error }));
|
||||
while (finished < count) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.ok(active <= 8); pending.splice(0).reverse().forEach(resolve => resolve());
|
||||
}
|
||||
const result = await completion, last = records.at(-1);
|
||||
assert.equal(maximum, 8); assert.equal(active, 0); assert.equal(last.attempts.length, count);
|
||||
assert.ok(last.attempts.every(attempt => attempt.status !== 'pending'));
|
||||
assert.equal(last.complete, !fail);
|
||||
if (fail) { assert.match(result.error.message, /setup failed/); assert.equal(last.attempts.filter(value => value.status === 'rejected').length, 1); }
|
||||
else assert.deepEqual(result.value.map(agent => agent.id), Array.from({ length: count }, (_, index) => `agent-${index}`));
|
||||
});
|
||||
|
||||
test('first successful observation uses only the original wait with no collection delay', async () => {
|
||||
const calls = [];
|
||||
const value = await observeStreaming({ readCount: async () => 16, target: 16, now: () => 1000,
|
||||
poll: async (read, milliseconds, target) => { calls.push(milliseconds); assert.ok(await read() >= target); } });
|
||||
assert.deepEqual(calls, [120000]); assert.equal(value.original.status, 'passed');
|
||||
assert.equal(value.observation.complete, true); assert.equal(value.observation.completedAtMs, 1000);
|
||||
});
|
||||
|
||||
test('known120s miss assessed before its nominal deadline stays failed and uses only the remaining600s observation', async () => {
|
||||
let clock = 1000, count = 30; const calls = [], records = [];
|
||||
const value = await observeStreaming({ readCount: async () => count, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => {
|
||||
calls.push(milliseconds); await read();
|
||||
if (calls.length === 1) { clock += milliseconds - 750; throw timeout(milliseconds); }
|
||||
count = 512; clock += 90000; await read();
|
||||
} });
|
||||
assert.deepEqual(calls, [120000, 480750]);
|
||||
assert.equal(value.original.status, 'failed'); assert.equal(value.original.lastObservedCount, 30);
|
||||
assert.equal(value.original.lastObservedAtMs, 1000); assert.equal(value.original.deadlineMs, 121000);
|
||||
assert.equal(value.original.assessedAtMs, 120250); assert.ok(value.original.assessedAtMs < value.original.deadlineMs);
|
||||
assert.equal(value.observation.completedAtMs, 210250); assert.equal(records[0].observation.complete, false);
|
||||
assert.equal(value.original.reason, 'original_streaming_deadline');
|
||||
});
|
||||
|
||||
test('page read failures and unknown poll failures never become an expected SLA miss', async () => {
|
||||
const failure = new Error('page detached');
|
||||
await assert.rejects(observeStreaming({ readCount: async () => { throw failure; }, target: 1,
|
||||
poll: async read => { try { await read(); } catch {} throw timeout(120000); } }), error => error === failure);
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 0, target: 1,
|
||||
poll: async () => { throw new Error('unrelated assertion'); } }), /Only the original streaming matcher deadline/);
|
||||
});
|
||||
|
||||
test('observation ceiling leaves a censored failure, never a substituted latency', async () => {
|
||||
let clock = 1000, calls = 0; const records = [];
|
||||
await assert.rejects(observeStreaming({ readCount: async () => 30, target: 512, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
poll: async (read, milliseconds) => { await read(); calls++; clock += milliseconds; throw timeout(milliseconds); } }), /Timeout/);
|
||||
assert.equal(calls, 2); assert.equal(clock, 601000);
|
||||
assert.equal(records.at(-1).observation.complete, false); assert.equal(records.at(-1).observation.censored, true);
|
||||
assert.equal(records.at(-1).observation.completedAtMs, null);
|
||||
});
|
||||
|
||||
test('original terminal is not reached after120s miss; observational180s is distinct', () => {
|
||||
const terminal = { complete: true, startedAtMs: 211000, deadlineMs: 391000, assessedAtMs: 212000 };
|
||||
assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), {
|
||||
status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null });
|
||||
assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed');
|
||||
assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
});
|
||||
|
||||
test('cursor inherits only paired exact known misses; final candidate inherits none', () => {
|
||||
const base = { misses: ['burst512:streaming120'] };
|
||||
requireH5Candidate('cursor-nonregression', base, base);
|
||||
const slowInvocation = { misses: ['invocation:observed-total-duration900'] };
|
||||
requireH5Candidate('cursor-nonregression', slowInvocation, slowInvocation);
|
||||
requireH5Candidate('final-b2', base, { misses: [] });
|
||||
for (const candidate of [{ misses: ['paced:streaming120'] }, { misses: ['invocation:observed-total-duration900'] },
|
||||
{ misses: ['operator-stop:streaming120'] }]) assert.throws(() => requireH5Candidate('cursor-nonregression', base, candidate));
|
||||
assert.throws(() => requireH5Candidate('final-b2', base, base), /every original SLA/);
|
||||
assert.throws(() => requireH5Candidate('final-b2', slowInvocation, slowInvocation), /every original SLA/);
|
||||
});
|
||||
@@ -1,10 +1,13 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS,
|
||||
originalTerminal, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
|
||||
test("four pairs must individually improve beyond noise", () => {
|
||||
@@ -77,17 +80,23 @@ const workloads = [
|
||||
function browserReport(revision, latency) {
|
||||
const companyId = "00000000-0000-4000-8000-000000000001", environmentId = "00000000-0000-4000-8000-000000000002";
|
||||
const localEnvironmentId = "00000000-0000-4000-8000-000000000003", controlAgentId = "00000000-0000-4000-8000-000000000004";
|
||||
const remoteAgentId = "00000000-0000-4000-8000-000000000005";
|
||||
const id = (kind, caseIndex, index) => `00000000-0000-4000-8000-${(kind * 100000 + caseIndex * 1000 + index).toString(16).padStart(12, "0")}`;
|
||||
const remoteAgentIds = Array.from({ length: 100 }, (_, index) => id(9, 0, index));
|
||||
const transportPolicy = { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" };
|
||||
return { protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport: "sandbox-remote-with-local-stop", transportPolicy, companyId,
|
||||
return { protocolVersion: H4D_PROTOCOL, transport: "sandbox-remote-with-local-stop", transportPolicy, companyId,
|
||||
bodyStartedAtMs: 1, agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: 100 },
|
||||
harnessProvenance: { sourceRevision: "b".repeat(40), sourceStatus: "", driverLockSha256: "8".repeat(64),
|
||||
sourceHashes: { "native-event-ingestion.journey.ts": "9".repeat(64), "fixtures/native-ingestion-h5.mjs": "a".repeat(64) } },
|
||||
resourceBudgets: { providerReadinessMs: 300000, fullInvocationMs: 900000, streamingMs: 120000,
|
||||
terminalDrainageMs: 180000, stopAcknowledgementMs: 30000, remoteLeaseCleanupMs: 30000 },
|
||||
local: { transport: "local", environmentId: localEnvironmentId, controlAgentId,
|
||||
selection: "Normal local environment + separate control agent defaultEnvironmentId; existing test-only Codex leaf command fixture",
|
||||
commandSha256: "6".repeat(64), loaderSha256: "7".repeat(64) },
|
||||
remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId, agentIds: [remoteAgentId],
|
||||
remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId, agentIds: remoteAgentIds,
|
||||
agentSetup: { count: 100, concurrency: 8, complete: true, startedAtMs: 2, settledAtMs: 40,
|
||||
attempts: remoteAgentIds.map((agentId, index) => ({ index, agentId, status: "fulfilled",
|
||||
startedAtMs: 3 + Math.floor(index / 8) * 2, settledAtMs: 4 + Math.floor(index / 8) * 2 })) },
|
||||
runnerPublicUrl: "wss://127.0.0.1:32100",
|
||||
selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override",
|
||||
plugin: { pluginKey: "paperclip.fake-sandbox-provider", assetHashes: Object.fromEntries(["worker.js", "plugin.js", "manifest.js", "lease-cleanup.js"].map(file => [file, "5".repeat(64)])) },
|
||||
@@ -104,7 +113,7 @@ function browserReport(revision, latency) {
|
||||
observedReadOnly: "on", observedIsolation: "repeatable read", kind: stop ? "host-file-journal" : "database-controller-checkpoint" };
|
||||
const rows = Array.from({ length: workload.runs }, (_, index) => ({
|
||||
run_id: id(1, caseIndex, index), issue_id: id(2, caseIndex, index), company_id: companyId,
|
||||
agent_id: stop ? controlAgentId : remoteAgentId,
|
||||
agent_id: stop ? controlAgentId : remoteAgentIds[index],
|
||||
lease_id: id(3, caseIndex, index), environment_id: actualEnvironmentId, runtime_mode: "native", provider: stop ? "local" : "fake-plugin",
|
||||
provider_lease_id: stop ? null : `fake-plugin://${id(1, caseIndex, index)}/fixture`,
|
||||
checkpoint_part: stop ? null : "header", checkpoint_header_bytes: stop ? null : 256,
|
||||
@@ -161,6 +170,224 @@ function browserReport(revision, latency) {
|
||||
}) };
|
||||
}
|
||||
|
||||
function h5Report(revision, loadedLatency) {
|
||||
const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000);
|
||||
Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE,
|
||||
workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true,
|
||||
actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture",
|
||||
serverEntrySha256: "e".repeat(64), compiledHotPathHashes: {
|
||||
"server/dist/services/heartbeat-run-events.js": "f".repeat(64),
|
||||
"server/dist/services/native-runtime/paperclip-control-plane-port.js": "1".repeat(64),
|
||||
"server/dist/vendor/paperclip-runner/native-session-runtime.js": "2".repeat(64) } });
|
||||
for (const item of report.cases) {
|
||||
const latency = item.name === "burst512" ? loadedLatency : 1000;
|
||||
item.settledAtMs = item.epochMs + latency; item.drainThroughTerminalMs = latency;
|
||||
item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency };
|
||||
item.intendedToVisible = { p95Ms: latency };
|
||||
if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1;
|
||||
item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs }));
|
||||
const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs;
|
||||
const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null,
|
||||
startedAtMs: item.epochMs, deadlineMs: item.epochMs + 120000, assessedAtMs,
|
||||
target: item.stop ? 3 : item.events, lastObservedCount: missed ? 30 : item.visible.length,
|
||||
lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs,
|
||||
error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null },
|
||||
observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } };
|
||||
const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000,
|
||||
assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs,
|
||||
lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false };
|
||||
item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation),
|
||||
visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length,
|
||||
independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10,
|
||||
deltaCount: item.runs * (item.stop ? 3 : item.events), resultAcceptedCount: item.stop ? 0 : item.runs, clockStreamsClosed: true } };
|
||||
item.terminalStatuses = item.runIds.map(runId => ({ runId, runtimeMode: "native", status: item.stop ? "cancelled" : "succeeded" }));
|
||||
for (const row of item.durable) row.resultAcceptedEvents = item.stop ? 0 : 1;
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
test("H5 assessor retains an original SLA failure assessed before its nominal streaming deadline", () => {
|
||||
const report = h5Report("B0", 180000), streaming = report.cases[3].h5.streaming.original;
|
||||
streaming.assessedAtMs = streaming.deadlineMs - 750;
|
||||
streaming.lastObservedAtMs = streaming.assessedAtMs - 1;
|
||||
const outcome = assessH5Arm(report, { status: "passed", retry: 0, errors: [], timeoutMs: H5_LIMITS.fullInvocationMs,
|
||||
startedAt: "2026-11-03T00:00:00.000Z", durationMs: 500000 });
|
||||
assert.equal(outcome.originalSlaPassed, false);
|
||||
assert.deepEqual(outcome.misses, ["burst512:streaming120"]);
|
||||
assert.throws(() => requireH5Candidate("final-b2", outcome, outcome), /every original SLA/);
|
||||
});
|
||||
|
||||
test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires complete owning evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; };
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" });
|
||||
const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map();
|
||||
async function inventory(report, playwrightFile, name) {
|
||||
const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision];
|
||||
if (!inventories.has(variant)) {
|
||||
const roots = ["server/dist", "ui/dist", "packages/db/src", "packages/shared/src"];
|
||||
const files = Object.entries({ ...report.compiledHotPathHashes, "server/dist/index.js": report.serverEntrySha256,
|
||||
"packages/db/src/client.ts": "3".repeat(64), "packages/shared/src/index.ts": "4".repeat(64), "ui/dist/index.html": "5".repeat(64) })
|
||||
.map(([path, sha256]) => ({ bytes: 10, path, sha256 })).sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
const aggregateSha256 = hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join("\n") + "\n");
|
||||
const manifestBytes = gzipSync(Buffer.from(JSON.stringify({ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, roots, fileCount: files.length, files, aggregateSha256 })));
|
||||
const manifestFile = path.join(directory, `${variant}-artifact-inventory.json.gz`); await writeFile(manifestFile, manifestBytes);
|
||||
const manifestFileSha256 = hash(manifestBytes);
|
||||
const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: {
|
||||
manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } });
|
||||
const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision,
|
||||
compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40),
|
||||
binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }],
|
||||
artifactInventorySummary: inventorySummaryFile });
|
||||
inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile,
|
||||
inventorySummarySha256: hash(await readFile(inventorySummaryFile)), qualifiedReceiptFile,
|
||||
qualifiedReceiptSha256: hash(await readFile(qualifiedReceiptFile)) });
|
||||
}
|
||||
return save(`${name}-inventory.json`, { schema: "paperclip.native-ingestion.arm-inventory.v1", variant,
|
||||
sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision,
|
||||
playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) });
|
||||
}
|
||||
async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) {
|
||||
const entries = [];
|
||||
for (let index = 0; index < 5; index++) {
|
||||
const entry = {};
|
||||
for (const arm of ["baseline", "candidate"]) {
|
||||
const report = h5Report(arm === "baseline" || sameRevision ? baselineRevision : candidateRevision, arm === "baseline" ? 180000 : candidateLatency);
|
||||
const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0,
|
||||
duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] };
|
||||
if (index === 0 && arm === "candidate") mutate?.(report, owner);
|
||||
owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json",
|
||||
body: Buffer.from(JSON.stringify(report)).toString("base64") });
|
||||
entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report);
|
||||
entry[`${arm}PlaywrightFile`] = await save(`${prefix}-${index}-${arm}-playwright.json`, { suites: [{ tests: [owner] }] });
|
||||
entry[`${arm}InventoryFile`] = await inventory(report, entry[`${arm}PlaywrightFile`], `${prefix}-${index}-${arm}`);
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
return save(`${prefix}-pairs.json`, entries);
|
||||
}
|
||||
try {
|
||||
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
|
||||
proof.protocolVersion = H4D_PROTOCOL;
|
||||
proof.measurementSha256 = "h5-synthetic-fixed-fixture";
|
||||
// H4d defaults use the same frozen source/geometry as the observation arms.
|
||||
for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field];
|
||||
for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; }
|
||||
const originalProof = await save("original-h4d-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{
|
||||
status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{
|
||||
name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] });
|
||||
const originalInventory = await inventory(proof, originalProof, "original-h4c");
|
||||
const finalProofArgs = [`--original-h4d-proof=${originalProof}`, `--original-h4d-inventory=${originalInventory}`];
|
||||
const noise = path.join(directory, "aa.json");
|
||||
const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]);
|
||||
assert.equal(aa.status, 0, aa.stderr);
|
||||
const aaReport = JSON.parse(await readFile(noise, "utf8"));
|
||||
assert.equal(aaReport.h5Assessments.length, 5);
|
||||
assert.deepEqual(aaReport.h5Assessments[0].baseline.misses, ["burst512:streaming120"]);
|
||||
const cursor = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor", 180000, { candidateRevision: "B1" })}`, `--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor.status, 0, cursor.stderr);
|
||||
const cursor900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-matched900", 180000, { candidateRevision: "B1", duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.equal(cursor900.status, 0, cursor900.stderr);
|
||||
const added900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression",
|
||||
`--pairs=${await manifest("cursor-added900", 180000, { candidateRevision: "B1", mutate: (_report, owner) => { owner.results[0].duration = 950000; } })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(added900.status, 0, "Cursor candidate cannot add an unmatched900 miss");
|
||||
const goodPairs = await manifest("good", 1000);
|
||||
const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(good.status, 0, good.stderr);
|
||||
const wrongPolicy = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", `--pairs=${goodPairs}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(wrongPolicy.status, 0); assert.match(wrongPolicy.stderr, /Cursor policy requires B0→B1/);
|
||||
const b1Noise = path.join(directory, "aa-b1.json");
|
||||
const b1AA = run(["--phase=aa", `--pairs=${await manifest("aa-b1", 181000, { sameRevision: true, baselineRevision: "B1" })}`, `--output=${b1Noise}`]);
|
||||
assert.equal(b1AA.status, 0, b1AA.stderr);
|
||||
const incremental = run(["--candidate-policy=final-b2", `--pairs=${await manifest("b1-b2", 1000, { baselineRevision: "B1" })}`,
|
||||
`--noise-report=${b1Noise}`, ...finalProofArgs, "--require-acceptance"]);
|
||||
assert.equal(incremental.status, 0, incremental.stderr);
|
||||
const pendingFile = path.join(directory, "pending-h4c.json");
|
||||
const pending = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, `--output=${pendingFile}`, "--require-acceptance"]);
|
||||
assert.notEqual(pending.status, 0);
|
||||
const pendingReport = JSON.parse(await readFile(pendingFile, "utf8"));
|
||||
assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false);
|
||||
const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.equal(performanceOnly.status, 0, performanceOnly.stderr);
|
||||
for (const [name, change, message] of [
|
||||
["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/],
|
||||
["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/],
|
||||
["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /H4d default deadlines/],
|
||||
["old-h4c-proof", value => { value.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream";
|
||||
value.harnessProvenance.sourceRevision = "7d7e7a88d562a95a1d407b76474763b666c181f5"; }, /Old H4c geometry/],
|
||||
["h4d-wrong-harness", value => { value.harnessProvenance.sourceRevision = "d".repeat(40); }, /same frozen harness source/],
|
||||
["h4d-wrong-measurement", value => { value.measurementSha256 = "old-h5a"; }, /same measured harness bytes/],
|
||||
["h4d-shared-agent", value => { value.cases[3].controllerAuthority.rows[1].agent_id = value.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/],
|
||||
["h4d-missing-stop-proof", value => { delete value.cases[5].localStopProof; }, /physical Stop/],
|
||||
["h4d-provider-still-alive", value => { value.cases[5].localStopProof.processes.providerAbsent = false; }, /provider remains alive/],
|
||||
["h4d-pending-local-lease", value => { value.cases[5].localStopProof.lease.status = "active"; }, /lease cleanup remains unsettled/],
|
||||
["h4d-wrong-local-lease", value => { value.cases[5].localStopProof.lease.id = value.cases[3].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4d-ack-timing-mismatch", value => { value.cases[5].stopAcknowledgedAtMs++; }, /acknowledgement timing boundary/],
|
||||
["h4d-missing-remote-leases", value => { delete value.cases[3].remoteLeases; }, /post-timing lease cleanup proof/],
|
||||
["h4d-pending-remote-lease", value => { value.cases[3].remoteLeases[0].cleanupStatus = "pending"; }, /lease cleanup remains unsettled/],
|
||||
["h4d-wrong-remote-lease", value => { value.cases[3].remoteLeases[0].id = value.cases[5].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4d-premature-remote-cleanup-proof", value => { value.cases[3].remoteLeasesCheckedAtMs = value.cases[3].settledAtMs - 1; }, /leases after recorded timing/],
|
||||
["h4d-wrong-authority-company", value => { value.cases[3].controllerAuthority.rows[0].company_id = "00000000-0000-4000-8000-000000099999"; }, /authority company mismatch/],
|
||||
["h4d-wrong-authority-run", value => { value.cases[3].controllerAuthority.rows[0].run_id = value.cases[5].runIds[0]; }, /authority run set mismatch/],
|
||||
]) {
|
||||
const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0],
|
||||
value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString());
|
||||
change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64");
|
||||
const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`);
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
|
||||
`--original-h4d-proof=${file}`, `--original-h4d-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]);
|
||||
assert.notEqual(attempt.status, 0, name);
|
||||
const rejected = JSON.parse(await readFile(output, "utf8"));
|
||||
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
|
||||
assert.match(rejected.originalH4dConfirmation.reason, message);
|
||||
}
|
||||
const badOutput = path.join(directory, "still-misses.json");
|
||||
const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`,
|
||||
`--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]);
|
||||
assert.notEqual(bad.status, 0);
|
||||
const rejected = JSON.parse(await readFile(badOutput, "utf8"));
|
||||
assert.equal(rejected.acceptancePassed, false);
|
||||
assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false);
|
||||
for (const [name, mutate, message] of [
|
||||
["old-h5a-protocol", report => { report.protocolVersion = "v5-complete-observation-original-sla"; }, /strictly equal/],
|
||||
["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/],
|
||||
["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/],
|
||||
["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/],
|
||||
["missing-delta", report => { report.cases[3].durable[99].individualDeltas--; }, /individualDeltas|result.events|512/],
|
||||
["wrong-workload", report => { report.workloadSha256 = "old"; }, /registration changed/],
|
||||
["pw-retry", (_report, owner) => { owner.results[0].retry = 1; }, /Retries/],
|
||||
["pw-error", (_report, owner) => { owner.results[0].errors.push({ message: "unexpected API failure" }); }, /deep-equal/],
|
||||
["wrong-observation-timeout", (_report, owner) => { owner.timeout = 900000; }, /1200000/],
|
||||
]) {
|
||||
const result = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutate })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(result.status, 0, name); assert.match(result.stderr, message, name);
|
||||
}
|
||||
const outer = run(["--candidate-policy=final-b2", `--pairs=${await manifest("outer900", 1000, { duration: 950000 })}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(outer.status, 0, "Observed total duration900 miss cannot qualify final B2");
|
||||
const incomplete = JSON.parse(await readFile(await manifest("missing-owner", 1000), "utf8"));
|
||||
delete incomplete[0].candidatePlaywrightFile;
|
||||
const missing = run(["--candidate-policy=final-b2", `--pairs=${await save("missing-owner-manifest.json", incomplete)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(missing.status, 0); assert.match(missing.stderr, /independent owning Playwright/);
|
||||
const global = JSON.parse(await readFile(await manifest("global-error", 1000), "utf8"));
|
||||
const globalOwner = JSON.parse(await readFile(global[0].candidatePlaywrightFile, "utf8"));
|
||||
globalOwner.errors = [{ message: "global teardown failed" }];
|
||||
await writeFile(global[0].candidatePlaywrightFile, JSON.stringify(globalOwner));
|
||||
const globalFailed = run(["--candidate-policy=final-b2", `--pairs=${await save("global-error-manifest.json", global)}`,
|
||||
`--noise-report=${noise}`, "--require-acceptance"]);
|
||||
assert.notEqual(globalFailed.status, 0); assert.match(globalFailed.stderr, /global error/);
|
||||
} finally { await rm(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
@@ -201,9 +428,18 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi
|
||||
["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/],
|
||||
["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/],
|
||||
["missing-retained-prelude", report => { report.cases.find(row => row.prelude).preludeProof.individualDeltas--; }, /stored prelude deltas/],
|
||||
["legacy-protocol", report => { delete report.protocolVersion; }, /v4 mixed clock protocol/],
|
||||
["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /v4 mixed clock protocol/],
|
||||
["old-remote-protocol", report => { report.protocolVersion = "v3-supported-remote-original-clock-stream"; }, /v4 mixed clock protocol/],
|
||||
["legacy-protocol", report => { delete report.protocolVersion; }, /independent-agent/],
|
||||
["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /independent-agent/],
|
||||
["old-remote-protocol", report => { report.protocolVersion = "v3-supported-remote-original-clock-stream"; }, /independent-agent/],
|
||||
["old-h4c-geometry", report => { report.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream"; }, /independent-agent/],
|
||||
["shared-participant-agent", report => { report.cases[3].controllerAuthority.rows[1].agent_id = report.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/],
|
||||
["participant-outside-pool", report => { report.cases[3].controllerAuthority.rows[1].agent_id = "00000000-0000-4000-8000-000000099999"; }, /outside the normal agent pool/],
|
||||
["incomplete-agent-pool", report => { report.remote.agentIds.pop(); }, /Incomplete independent/],
|
||||
["unfinished-agent-setup", report => { report.remote.agentSetup.complete = false; }, /settle successfully/],
|
||||
["rejected-agent-setup", report => { report.remote.agentSetup.attempts[0].status = "rejected"; }, /Rejected agent setup/],
|
||||
["agent-setup-concurrency", report => { report.remote.agentSetup.concurrency = 9; }, /strictly equal/],
|
||||
["agent-setup-overlap", report => { for (const row of report.remote.agentSetup.attempts.slice(0, 9)) { row.startedAtMs = 3; row.settledAtMs = 39; } }, /exceeded the fixed concurrency/],
|
||||
["late-agent-setup", report => { report.remote.agentSetup.settledAtMs = 60; }, /before every case activation/],
|
||||
["wrong-transport", report => { report.transport = "local"; }, /original local Stop transport/],
|
||||
["missing-transport-policy", report => { delete report.transportPolicy; }, /explicit transport/],
|
||||
["fake-remote-stop", report => { report.cases[5].transport = "sandbox-remote"; }, /Wrong per-case transport/],
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# Frozen H5b application helpers
|
||||
|
||||
These are exact source snapshots of the four reviewed task helpers. `sources.json`
|
||||
records their original paths, SHA256 values and the frozen inputs. No helper code
|
||||
or embedded path was changed for packaging. The owning run receipt identifies the bytes
|
||||
that actually executed. Copying tools into a delivery commit does not rerun or
|
||||
change the provenance of earlier evidence.
|
||||
|
||||
The recorded SMALL post-read check used extractor SHA256
|
||||
`00930574798e3d56257a4913d82400ccd739b4bb52e9a24d5cb18a36892be970`
|
||||
(5445 bytes), retained with its raw 23-file artifact manifest. The packaged
|
||||
extractor is the later reviewed SHA256
|
||||
`a052375a80b3abad4513cdf019325e83fed63c67d935f1dc2accba753ce589f0`
|
||||
(5704 bytes). It adds three existing-history prelude assertions: foreground run
|
||||
identity, 1000 unique deltas, and verification before epoch with retained head.
|
||||
It did not produce the SMALL proof. Measured driver bytes are unchanged.
|
||||
|
||||
The executed driver is tag `native-ingestion-h5b`, annotated tag object
|
||||
`34498d7e09b61493e0f6703c354291c485836298`, commit
|
||||
`179fe6fa6da1b24ca08c1162301538d3e3788af6`. Its lock SHA256 is
|
||||
`54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8`.
|
||||
The delivery lock is different. Do not replace it or describe a delivery checkout
|
||||
as that executed driver. The [frozen registration](../../../tests/e2e/fixtures/native-ingestion-h5.md)
|
||||
sets the independent-agent geometry, deadlines and evidence gates.
|
||||
|
||||
## Recorded Linux setup
|
||||
|
||||
Use a disposable, quiet Linux host with the recorded Node/pnpm, PostgreSQL,
|
||||
Chromium and native Runner inputs. Install and normally build each exact APP
|
||||
revision independently; retain its build receipt and all four inventory roots.
|
||||
Keep source, node_modules, TypeScript output and compiled APP output separate.
|
||||
Only the native Rust target could be reused: all 15 native inputs matched exactly,
|
||||
including Rust tree `bbe29bba574ce8d5cfddf2f7461ae0eb5983df07`.
|
||||
The build receipts do not imply three fresh Rust compilations. The three APP
|
||||
revisions and directory names are in `sources.json`. Raw B2 `ca7cba1` is the unoptimized
|
||||
batching base. A future Rust candidate needs a new immutable revision, normal
|
||||
build and qualified inventory; none is supplied here.
|
||||
|
||||
Keep the recorded layout under `/home/ubuntu/native-ingestion`: the independent
|
||||
179 driver at `harness-h5b`, APP trees at `aligned-baseline`, `aligned-cursors`
|
||||
and `aligned-batching`, qualified build receipts and inventories in their recorded
|
||||
locations, and these four helpers in `helpers` under their original basenames.
|
||||
Verify their copied bytes against `sources.json`. The qualification/readiness
|
||||
wrappers intentionally retain those paths. Any path-portable derivative must
|
||||
record its changes and separate hashes.
|
||||
|
||||
Run this snippet from the current tools checkout. The paths retain the recorded
|
||||
Linux layout. A new driver/output path must be absent before creation.
|
||||
|
||||
```sh
|
||||
TOOLS_CHECKOUT="$(git rev-parse --show-toplevel)"
|
||||
DRIVER_ROOT=/home/ubuntu/native-ingestion/harness-h5b
|
||||
HELPERS_ROOT=/home/ubuntu/native-ingestion/helpers
|
||||
git -C "$TOOLS_CHECKOUT" show-ref --verify --quiet refs/tags/native-ingestion-h5b ||
|
||||
git -C "$TOOLS_CHECKOUT" fetch origin tag native-ingestion-h5b
|
||||
test "$(git -C "$TOOLS_CHECKOUT" rev-parse 'native-ingestion-h5b^{commit}')" = 179fe6fa6da1b24ca08c1162301538d3e3788af6
|
||||
git -C "$TOOLS_CHECKOUT" worktree add --detach "$DRIVER_ROOT" native-ingestion-h5b
|
||||
pnpm --dir "$DRIVER_ROOT" install --frozen-lockfile --offline --ignore-scripts
|
||||
mkdir -p "$HELPERS_ROOT"
|
||||
for helper in native-ingestion-run-h5b-browser.mjs native-ingestion-linux-h5b-qualification.py native-ingestion-h5b-linux-readiness.mjs native-ingestion-extract-h5b-qualification.py sources.json; do
|
||||
cp "$TOOLS_CHECKOUT/scripts/native-ingestion/h5b/$helper" "$HELPERS_ROOT/$helper"
|
||||
done
|
||||
node "$DRIVER_ROOT/scripts/describe-native-ingestion-h5.mjs"
|
||||
node "$TOOLS_CHECKOUT/scripts/native-ingestion/h5b/native-ingestion-run-h5b-browser.mjs"
|
||||
```
|
||||
|
||||
The last two commands are dry runs. They start no app or performance arm.
|
||||
The recorded driver install is independent of each APP's normal install/build.
|
||||
Use the original frozen driver for active comparisons and later same-source H4d
|
||||
confirmation, even when these source bytes also appear in the delivery PR.
|
||||
|
||||
## Qualification and comparisons
|
||||
|
||||
Only start one released stage in an exclusive quiet window. Unset inherited
|
||||
`DATABASE_URL`, `PAPERCLIP_RUNNER_STATE_DIR` and diagnostic overrides as the wrapper does.
|
||||
Use fresh output names; preserve failures and existing output. After the build
|
||||
and live-inventory checks, the recorded qualification commands are:
|
||||
|
||||
```sh
|
||||
QUALIFICATION_OUTPUT=/home/ubuntu/native-ingestion/evidence/h5b/qualification/NEW-FULL
|
||||
python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-linux-h5b-qualification.py --tree=B0 --name=NEW-SMALL
|
||||
python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-linux-h5b-qualification.py --tree=B0 --full --name=NEW-FULL
|
||||
python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-extract-h5b-qualification.py "$QUALIFICATION_OUTPUT" --revision=bac53ec3bfeaf6546e7aaeaee516487b30534654 --full
|
||||
```
|
||||
|
||||
The extractor's `--full` must match the qualification. It verifies retained
|
||||
reports and emits qualification-only output. The readiness helper additionally
|
||||
checks actual B0/B1 inventories, native target, PATH and physical quiet; its
|
||||
output path must be fresh. It does not start an APP.
|
||||
|
||||
The browser orchestrator prints its five stages without arguments. A stage needs
|
||||
an explicit byte-frozen `--release=...`, `--stage=...`, `--output-root=...` and
|
||||
`--execute`. Static `--validate-release` checks do not execute an arm. Release
|
||||
inputs bind the exact driver, tools, APP/build inventories, stage order and prior
|
||||
noise reports. Qualify full B0 and forecast duration before fresh B0 and B1 A/A;
|
||||
freeze both noise results before implementing another product optimization.
|
||||
|
||||
H5b uses 100 normal independent agents, one concurrent run each, and eight setup
|
||||
workers. Counts, rates and original polls stay fixed. H5 observation has explicit
|
||||
600s streaming and 1200s invocation limits. The original 120/180/900s outcomes stay
|
||||
separate; final B2 must meet every original SLA in every arm and later pass H4d
|
||||
on the same frozen source/geometry/SUT. The 30-minute resource watchdog grants no
|
||||
SLA extension or performance acceptance.
|
||||
|
||||
Status at the recorded 2026-11-04 05:27:45 UTC launch: B0 SMALL passed functional
|
||||
qualification; full B0 had started. The later full B0 H5b attempt **failed**:
|
||||
83 runs succeeded and 17 failed; owning Playwright duration was 742.825s.
|
||||
Existing-history and Stop were not reached. Authority checks verified 100 distinct
|
||||
agents, and source clocks verified all 51,200 emissions. Complete canonical
|
||||
persisted counts are not yet established. Full PostgreSQL causal forensics remain
|
||||
pending. This is not a passed FULL qualification or an accepted performance arm.
|
||||
|
||||
This source package claims no A/A, browser gain or application acceptance.
|
||||
The old 2-agent × 50-run failure remains a product limitation. Current run receipts
|
||||
and the dated benchmark report, not this source package, establish later outcomes.
|
||||
@@ -0,0 +1,63 @@
|
||||
import pathlib,json,base64,argparse,hashlib
|
||||
p=argparse.ArgumentParser();p.add_argument('directory');p.add_argument('--revision',required=True);p.add_argument('--full',action='store_true');a=p.parse_args();d=pathlib.Path(a.directory);r=json.loads((d/'playwright.json').read_text());attachments=[]
|
||||
def visit(value):
|
||||
if isinstance(value,list):
|
||||
for child in value:visit(child)
|
||||
elif isinstance(value,dict):
|
||||
attachments.extend(item for item in value.get('attachments',[])if item.get('name')=='native-ingestion-evidence')
|
||||
for key,child in value.items():
|
||||
if key!='attachments':visit(child)
|
||||
visit(r);assert len(attachments)==1,len(attachments);item=attachments[0]
|
||||
raw=base64.b64decode(item['body'])if item.get('body')else pathlib.Path(item['path']).read_bytes();report=json.loads(raw);(d/'native-ingestion-evidence.json').write_bytes(raw)
|
||||
assert report.get('complete')is True and not report.get('primaryFailure'),report.get('primaryFailure')
|
||||
assert report['sourceRevision']==report['compiledBuildRevision']==a.revision
|
||||
assert report['protocolVersion']=='v5b-independent-agents-complete-observation-original-sla'
|
||||
assert report['observationMode']=='h5b-observe-original-sla-independent-agents' and report['observationComplete'] is True
|
||||
assert report['measurementSha256']=='452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e'
|
||||
assert report['transport']=='sandbox-remote-with-local-stop'
|
||||
assert report['admissionProtocol']=='two-at-a-time-provider-and-clock-readiness-before-common-epoch'
|
||||
assert report['setupActivationConcurrency']==2
|
||||
assert report['harnessProvenance']['sourceRevision']=='179fe6fa6da1b24ca08c1162301538d3e3788af6'
|
||||
assert report['qualificationOnly']is(not a.full)
|
||||
assert report['serverArtifact']=='compiled' and report.get('diagnosticOnly')is False
|
||||
assert report['cleanup']=={'errors':[],'complete':True}
|
||||
expected=['paced','burst16','burst128','burst512','existing-history','operator-stop']if a.full else['qualification','operator-stop'];assert [c['name']for c in report['cases']]==expected
|
||||
ids=report['remote']['agentIds'];setup=report['remote']['agentSetup']
|
||||
assert len(ids)==(100 if a.full else 3) and len(set(ids))==len(ids)
|
||||
assert setup['complete'] is True and setup['concurrency']==8 and setup['count']==len(ids)
|
||||
assert len(setup['attempts'])==len(ids) and all(x['status']=='fulfilled'for x in setup['attempts'])
|
||||
assert all(ids[x['index']]==x['agentId']for x in setup['attempts'])
|
||||
rows=[]
|
||||
for case in report['cases']:
|
||||
stop=case['name']=='operator-stop';assert case['persistedAfterReload']is True
|
||||
assert case['setupActivationConcurrency']==2 and case['participants']==case['runs']
|
||||
assert sorted(x['index']for x in case['activations'])==list(range(case['runs']))
|
||||
assert all(x['status']=='fulfilled'for x in case['activations'])
|
||||
waves=case['activationWaves'];assert len(waves)==(case['runs']//2)
|
||||
assert all(1<=len(w['indices'])<=2 and w['statuses']==['fulfilled']*len(w['indices'])for w in waves)
|
||||
if not a.full and not stop:assert(case['runs'],case['events'],case['intervalMs'])==(3,16,25)
|
||||
assert len(case['durable'])==case['runs']==len(case['runIds'])
|
||||
if case['name']=='existing-history':
|
||||
pre=case['preludeProof'];assert pre['runId']==case['runIds'][0] and pre['individualDeltas']==1000 and pre['uniqueSourceIdentities']==1000
|
||||
assert pre['verifiedAtMs']<case['epochMs'] and pre['retainedGlobalHead']>=1000
|
||||
assert case['matchedVisibleClocks']==len(case['visible']) and len(case['visible'])>=(3 if stop else case['events'])
|
||||
authority=case['controllerAuthority'];assert authority['kind']==('host-file-journal'if stop else'database-controller-checkpoint')
|
||||
assert authority['observedReadOnly']=='on'and authority['observedIsolation']=='repeatable read'
|
||||
assert len(authority['rows'])==case['runs']and authority['connectionClosedAtMs']<case['epochMs']
|
||||
for proof in case['durable']:
|
||||
assert proof['matchedSourceClocks']==proof['individualDeltas']
|
||||
assert proof['uniqueSourceIdentities']==proof['individualDeltas']
|
||||
if not stop:assert proof['individualDeltas']==case['events']
|
||||
if stop:
|
||||
s=case['localStopProof'];db=s['database']['rows'][0]
|
||||
assert db['status']=='cancelled'and db['cancellation_dispatch_state']=='acknowledged'
|
||||
assert all(s['processes'][k]is True for k in['runnerAbsent','groupAbsent','providerAbsent'])
|
||||
assert s['lease']['releasedAt']and s['lease']['cleanupStatus']!='failed'
|
||||
if not db['process_pid']and not db['process_group_id']:assert int(db['process_stopped_seq'])>int(db['process_identity_seq'])
|
||||
else:
|
||||
assert len(set(x['agent_id']for x in case['controllerAuthority']['rows']))==case['runs']
|
||||
assert all(x['agent_id']in ids for x in case['controllerAuthority']['rows'])
|
||||
assert len(case['remoteLeases'])==case['runs']
|
||||
assert all(lease['releasedAt']and lease['cleanupStatus']=='success'for lease in case['remoteLeases'])
|
||||
rows.append({'name':case['name'],'runs':case['runs'],'eventsPerRun':case['events'],'visibleMarkers':len(case['visible']),'individualDeltas':sum(v['individualDeltas']for v in case['durable']),'sourceClocksMatched':sum(v['matchedSourceClocks']for v in case['durable']),'runIds':case['runIds'],'transport':case['transport'],'setupWaves':len(case['activationWaves']),'setupMs':case['allReadyObservedAtMs']-case['setupReadinessStartedAtMs'],'drainThroughTerminalMs':case['drainThroughTerminalMs']})
|
||||
summary={'complete':True,'qualificationOnly':True,'performanceAcceptanceEligible':False,'sourceRevision':a.revision,'harnessRevision':report['harnessProvenance']['sourceRevision'],'reportSha256':hashlib.sha256(raw).hexdigest(),'fullFixture':a.full,'cases':rows,'cleanup':report['cleanup']};(d/'independent-qualification-check.json').write_text(json.dumps(summary,indent=2)+'\n');print(json.dumps(summary))
|
||||
@@ -0,0 +1,29 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {readFile,writeFile,realpath} from 'node:fs/promises';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
import {createHash} from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
const base='/home/ubuntu/native-ingestion',helper=base+'/helpers/native-ingestion-run-h5b-browser.mjs',out=base+'/evidence/h5b/setup';
|
||||
const sha=b=>createHash('sha256').update(b).digest('hex');
|
||||
assert.equal(sha(await readFile(helper)),'522d4fc718665ef484dec40ff56e463efe11b4ddf0c632c2d3f4811092d63fa6');
|
||||
const h=await import(helper);
|
||||
const quiet=await h.physicalQuietGuard({receiptFile:out+'/linux-quiet-readiness.json',phase:'pre-arm',armStage:'H5b-SETUP-ONLY'});
|
||||
const receiptFile=base+'/aligned-4a83-builds-qualified.json',summaryFile=base+'/evidence/aligned-4a83/environment/artifact-inventory-summary.json';
|
||||
const receipt=JSON.parse(await readFile(receiptFile,'utf8')),summary=JSON.parse(await readFile(summaryFile,'utf8'));
|
||||
const results=[];
|
||||
for(const [variant,directory] of [['B0','aligned-baseline'],['B1','aligned-cursors']]) {
|
||||
const row=receipt.results.find(v=>v.variant===variant),i=summary.inventories[variant],native=receipt.nativeStageHashes.find(v=>v.variant===variant);
|
||||
const descriptor={appRoot:base+'/'+directory,revision:row.revision,qualifiedReceiptFile:receiptFile,qualifiedReceiptSha256:sha(await readFile(receiptFile)),inventorySummaryFile:summaryFile,inventorySummarySha256:sha(await readFile(summaryFile)),manifestFile:path.join(path.dirname(summaryFile),i.manifest),manifestFileSha256:i.manifestSha256,aggregateSha256:i.aggregateSha256,binarySha256:native.binarySha256,rustSourceTree:receipt.rustSourceTree};
|
||||
const q=await h.validateVariantBuild(variant,descriptor),live=await h.validateActualInventory(descriptor,q.manifest),ui=await h.assertNoPreferredUiShadow(descriptor.appRoot);
|
||||
assert.equal(execFileSync('git',['rev-parse','HEAD'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),descriptor.revision);
|
||||
assert.equal(execFileSync('git',['status','--short'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),'');
|
||||
assert.equal(JSON.parse(await readFile(path.join(descriptor.appRoot,'server/dist/build-info.json'),'utf8')).commit,descriptor.revision);
|
||||
const target=path.join(descriptor.appRoot,'packages/paperclip-runner/runner/target'),binary=path.join(target,'release/paperclip-runnerd');
|
||||
assert.equal(await realpath(target),native.targetRealPath);assert.equal(sha(await readFile(binary)),native.binarySha256);
|
||||
results.push({variant,descriptor,live,ui,nativeTargetRealPath:await realpath(target)});
|
||||
}
|
||||
const pathNode=execFileSync('node',['-p','process.execPath'],{encoding:'utf8'}).trim();
|
||||
assert.equal(await realpath(pathNode),await realpath(process.execPath));
|
||||
const data={setupOnly:true,noSutBrowserProviderOrEvents:true,performanceAcceptanceEligible:false,atMs:Date.now(),helperSha256:sha(await readFile(helper)),harnessRevision:'179fe6fa6da1b24ca08c1162301538d3e3788af6',quiet,path:{PATH:process.env.PATH,nodeExecutable:process.execPath,pathResolvedNode:pathNode,nodeVersion:process.version},results};
|
||||
await writeFile(out+'/linux-readiness.json',JSON.stringify(data,null,2)+'\n',{flag:'wx'});
|
||||
console.log(JSON.stringify({complete:true,setupOnly:true,noSutBrowserProviderOrEvents:true,variants:results.map(v=>({variant:v.variant,fileCount:v.live.fileCount,aggregate:v.live.aggregateSha256,uiShadowAbsent:v.ui.preferredIndexAbsent})),quiet:quiet.passed,nodeVersion:process.version}));
|
||||
@@ -0,0 +1,58 @@
|
||||
import pathlib,subprocess,json,os,socket,datetime,hashlib,traceback,argparse
|
||||
p=argparse.ArgumentParser();p.add_argument('--full',action='store_true');p.add_argument('--tree',choices=['B0','B1','B2'],default='B2');p.add_argument('--name',required=True);a=p.parse_args()
|
||||
base=pathlib.Path('/home/ubuntu/native-ingestion');repo=base/'paperclip';h=base/'harness-h5b';out=base/'evidence/h5b/qualification'/a.name;out.mkdir(parents=True,exist_ok=False)
|
||||
pins={'B0':('aligned-baseline','bac53ec3bfeaf6546e7aaeaee516487b30534654'),'B1':('aligned-cursors','07061a601cdc20603c8c7a46f9ddbad7ac5f5939'),'B2':('aligned-batching','ca7cba171cd5198ed569d75012edc858041cbc44')};app=base/pins[a.tree][0]
|
||||
ref='179fe6fa6da1b24ca08c1162301538d3e3788af6';protocol='452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e';comparator='0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168';lock='54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8'
|
||||
r={'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'qualificationOnly':True,'performanceAcceptanceEligible':False,'fullFixture':a.full,'sourceRevision':pins[a.tree][1],'harnessRevision':ref,'steps':[]}
|
||||
def now():return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
def save():(out/'owner-receipt.json').write_text(json.dumps(r,indent=2)+'\n')
|
||||
def cmd(name,args,cwd,env=None,timeout=None):
|
||||
row={'name':name,'argv':list(map(str,args)),'cwd':str(cwd),'startedAt':now()};r['steps'].append(row);save()
|
||||
with(out/(name+'.log')).open('xb')as log:result=subprocess.run(args,cwd=cwd,env=env,stdout=log,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
row.update({'exit':result.returncode,'finishedAt':now()});save();assert result.returncode==0,(name,result.returncode)
|
||||
try:
|
||||
assert(base/'aligned-4a83-builds.exit').read_text().strip()=='0'
|
||||
qualified=json.loads((base/'aligned-4a83-builds-qualified.json').read_text())
|
||||
assert {row['variant']:row['revision'] for row in qualified['results']}=={v:pin for v,(directory,pin) in pins.items()}
|
||||
rows=[]
|
||||
for q in pathlib.Path('/proc').iterdir():
|
||||
if not q.name.isdigit():continue
|
||||
try:
|
||||
exe=os.readlink(q/'exe')
|
||||
if pathlib.Path(exe).name in['node','postgres','paperclip-runner','rustc','cargo','chrome','chromium','chrome-headless-shell']:rows.append({'pid':int(q.name),'executable':exe})
|
||||
except OSError:pass
|
||||
assert not rows,rows
|
||||
for directory,pin in pins.values():
|
||||
root=base/directory;assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=root,text=True).strip()==pin
|
||||
assert not subprocess.check_output(['git','status','--short'],cwd=root,text=True).strip()
|
||||
assert json.loads((root/'server/dist/build-info.json').read_text())['commit']==pin
|
||||
assert h.exists(), 'Stage the exact independent driver before qualification'
|
||||
assert not(h/'node_modules').is_symlink();assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=h,text=True).strip()==ref
|
||||
assert not subprocess.check_output(['git','status','--short'],cwd=h,text=True).strip()
|
||||
files=['native-event-ingestion.journey.ts','playwright-native-ingestion.config.ts','fixtures/in-feed-codex.mjs','fixtures/native-ingestion-bin/codex','fixtures/composer-stop-codex-loader.mjs','fixtures/native-ingestion-bootstrap.mjs','fixtures/native-ingestion-remote-server.mjs','fixtures/native-ingestion-protocol-qualification.mjs','fixtures/native-ingestion-h5.mjs']
|
||||
assert hashlib.sha256(b''.join((h/'tests/e2e'/f).read_bytes()for f in files)).hexdigest()==protocol
|
||||
assert hashlib.sha256((h/'scripts/compare-native-ingestion-browser.ts').read_bytes()).hexdigest()==comparator
|
||||
assert hashlib.sha256((h/'pnpm-lock.yaml').read_bytes()).hexdigest()==lock
|
||||
assert hashlib.sha256((h/'tests/e2e/fixtures/native-ingestion-h5.md').read_bytes()).hexdigest()=='60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54'
|
||||
r['resourceWatchdogMillis']=1800000;r['resourceWatchdogScope']='Resource-only including startup/test/teardown; original SLA and H5 observation unchanged';r['protocolSha256']=protocol;r['comparatorSha256']=comparator;r['driverLockSha256']=lock
|
||||
with socket.socket()as s:s.bind(('127.0.0.1',0));port=s.getsockname()[1]
|
||||
env=dict(os.environ)
|
||||
for key in ['DATABASE_URL','PAPERCLIP_RUNNER_STATE_DIR','PAPERCLIP_INGESTION_SERVER_COMMAND','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR','PAPERCLIP_INGESTION_BROWSER_CAPTURE','PAPERCLIP_INGESTION_SMOKE','NODE_OPTIONS','PAPERCLIP_INGESTION_OBSERVATION_MODE']:env.pop(key,None)
|
||||
binary=str(app/'packages/paperclip-runner/runner/target/release/paperclip-runnerd')
|
||||
overrides={'PAPERCLIP_INGESTION_OBSERVATION_MODE':'h5b-observe-original-sla-independent-agents','PAPERCLIP_INGESTION_TRANSPORT':'sandbox-remote-with-local-stop','PAPERCLIP_INGESTION_APP_ROOT':str(app),'PAPERCLIP_INGESTION_EVIDENCE_DIR':str(out),'PAPERCLIP_E2E_PORT':str(port),'PAPERCLIP_RUNNER_BINARY':binary,'PAPERCLIP_RUNNER_REMOTE_BINARY_PATH':binary,'PAPERCLIP_INGESTION_VARIANT':a.name+'-FUNCTIONAL-QUALIFICATION-ONLY','PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES':'Independent H5b frozen offline ignore-scripts driver install; application owns independent normal locked install/build'}
|
||||
if not a.full:overrides['PAPERCLIP_INGESTION_SMOKE']='1'
|
||||
env.update(overrides);r['environmentOverrides']=overrides;save();subprocess.run(['sync'],check=True)
|
||||
cmd('playwright',['timeout','--signal=INT','--kill-after=40s','30m','node','node_modules/@playwright/test/cli.js','test','--config','tests/e2e/playwright-native-ingestion.config.ts'],h,env,1845)
|
||||
r['playwrightPassed']=True;r['finishedAt']=now();save()
|
||||
except BaseException as error:r.update({'passed':False,'error':str(error),'traceback':traceback.format_exc(),'finishedAt':now()});save();(out/'owner.exit').write_text('1\n');raise
|
||||
|
||||
finally:
|
||||
helper=base/'helpers/native-ingestion-run-h5b-browser.mjs'
|
||||
guard="import net from 'node:net'; import {physicalQuietGuard} from '"+str(helper)+"'; const s=net.createServer(); await new Promise((ok,bad)=>{s.once('error',bad);s.listen(Number(process.argv[2]),'127.0.0.1',ok)}); await new Promise(ok=>s.close(ok)); await physicalQuietGuard({receiptFile:process.argv[1],phase:'post-arm',armStage:'H5b-QUALIFICATION-ONLY'});"
|
||||
try:
|
||||
result=subprocess.run(['node','--input-type=module','-e',guard,str(out/'physical-quiet-after.json'),str(r['environmentOverrides']['PAPERCLIP_E2E_PORT'])],check=True,capture_output=True,text=True)
|
||||
(out/'port-after.json').write_text(json.dumps({'complete':True,'port':r['environmentOverrides']['PAPERCLIP_E2E_PORT'],'checkedAt':now(),'nodeBindAndListenPassed':True})+'\n')
|
||||
if r.get('playwrightPassed'):
|
||||
r['passed']=True;r['cleanupComplete']=True;save();(out/'owner.exit').write_text('0\n')
|
||||
except BaseException as cleanup_error:
|
||||
r['externalQuietFailure']=str(cleanup_error);r['passed']=False;r['cleanupComplete']=False;save();(out/'owner.exit').write_text('1\n');raise
|
||||
@@ -0,0 +1,652 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, writeFile, mkdir, open, access, readdir, realpath, stat, lstat } from 'node:fs/promises';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import net from 'node:net';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
// Prospective H5 application orchestrator. No launch without --execute and a
|
||||
// byte-frozen root release. Original ce33 helper SHA is provenance only:
|
||||
// 2244e238642caafd488a7c86fb463376c8459ee0f06f8054555407a174698fa1.
|
||||
export const MODE = 'h5b-observe-original-sla-independent-agents';
|
||||
export const PROTOCOL = 'v5b-independent-agents-complete-observation-original-sla';
|
||||
export const WORKLOAD_SHA256 = '9d34b1511610cd5fcc73f8d734114e624105a3ee3b1e45c2b4d1cb721cad048c';
|
||||
const H5_REVISION = '179fe6fa6da1b24ca08c1162301538d3e3788af6';
|
||||
const OLD_NATIVE_TARGET = '/home/ubuntu/native-ingestion/aligned-baseline/packages/paperclip-runner/runner/target';
|
||||
const H5_PROTOCOL_SHA256 = '452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e';
|
||||
const H5_COMPARATOR_SHA256 = '0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168';
|
||||
const H5_INVENTORY_HELPER_SHA256 = '76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9';
|
||||
const H5_PROTOCOL_DOCUMENT_SHA256 = '60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54';
|
||||
const INVENTORY_ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
export const RESOURCE_WATCHDOG_MILLIS = 30 * 60_000;
|
||||
export const STAGES = Object.freeze(['browser-aa-b0', 'browser-aa-b1', 'browser-b0-b1', 'browser-b1-b2', 'browser-b0-b2']);
|
||||
export const FINALIZE_STAGE = 'browser-finalize-h4d';
|
||||
const protocolFiles = ['native-event-ingestion.journey.ts', 'playwright-native-ingestion.config.ts', 'fixtures/in-feed-codex.mjs',
|
||||
'fixtures/native-ingestion-bin/codex', 'fixtures/composer-stop-codex-loader.mjs', 'fixtures/native-ingestion-bootstrap.mjs',
|
||||
'fixtures/native-ingestion-remote-server.mjs', 'fixtures/native-ingestion-protocol-qualification.mjs', 'fixtures/native-ingestion-h5.mjs'];
|
||||
const BASELINES = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654', B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' });
|
||||
const OLD_B2 = 'ca7cba171cd5198ed569d75012edc858041cbc44';
|
||||
const OLD_RECEIPT = 'a1259ba760f3cbbe44817e333f9f620bcf3769fb0a6c16f9ce2f0f5b2ed60d94';
|
||||
const OLD_SUMMARY = 'd3766fc4206d569d10fd02224433f1e9a2caa6992a7c62216e80af368aa4e44d';
|
||||
const OLD_BINARY = 'abb65bdc073991e428bd37f9809603ee7564543cca0ed0305dc231137c93070f';
|
||||
const OLD_RUST = 'bbe29bba574ce8d5cfddf2f7461ae0eb5983df07';
|
||||
const OLD_INVENTORIES = Object.freeze({
|
||||
B0: { manifestFileSha256: 'a1c797a75dd3174f414fc5529acf56783e1a3dfe7b22f1e460a87b60f6bff01d', aggregateSha256: 'a50720be2ce53f9222187d63609c5936f94b6a00c60df2ca112261abd1bf921e' },
|
||||
B1: { manifestFileSha256: 'df48e5c511a2f31e31180e05f18e2322ca4f384ea06f6f96a57de24d39849ef8', aggregateSha256: 'bca5b6fad62e70631c135a9ff377efeb587aba17ef0842c8fd82804ed99a81ab' },
|
||||
});
|
||||
const LOCK_SHA256 = '54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8';
|
||||
const GATES_SHA256 = 'ded8bcbb8f05732348f6520b9c5e41b7c35bc098997eb263a9dcb174eede75fc';
|
||||
const selfFile = fileURLToPath(import.meta.url);
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const getJson = async file => JSON.parse(await readFile(file, 'utf8'));
|
||||
const saveJson = async (file, value, fresh = false) => writeFile(file, JSON.stringify(value, null, 2) + '\n', fresh ? { flag: 'wx' } : undefined);
|
||||
const digest = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{64}$/.test(value), `Missing exact ${name} SHA-256`);
|
||||
const revision = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{40}$/.test(value), `Missing immutable full ${name} revision`);
|
||||
const absolute = (value, name) => assert.ok(typeof value === 'string' && path.isAbsolute(value), `Missing absolute ${name} path`);
|
||||
const option = name => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
|
||||
export function stagePlan(stage) {
|
||||
assert.ok(STAGES.includes(stage) || stage === FINALIZE_STAGE, 'Select one registered H5 stage');
|
||||
if (stage === FINALIZE_STAGE) return { stage, requiredVariants: ['B0', 'B1', 'B2'], priorStages: [...STAGES], fullArms: 0, finalizationOnly: true };
|
||||
const [baseline, candidate] = stage === STAGES[0] ? ['B0', 'B0'] : stage === STAGES[1] ? ['B1', 'B1']
|
||||
: stage === STAGES[2] ? ['B0', 'B1'] : stage === STAGES[3] ? ['B1', 'B2'] : ['B0', 'B2'];
|
||||
const aa = baseline === candidate;
|
||||
return { stage, baseline, candidate, aa, expectation: stage === STAGES[2] ? 'nonregression' : 'improvement',
|
||||
candidatePolicy: aa ? null : stage === STAGES[2] ? 'cursor-nonregression' : 'final-b2',
|
||||
requiredVariants: [...new Set([baseline, candidate])], priorStages: STAGES.slice(0, STAGES.indexOf(stage)),
|
||||
pairs: 5, fullArms: 10, noiseStage: aa ? null : `browser-aa-${baseline.toLowerCase()}` };
|
||||
}
|
||||
|
||||
async function hashedJson(file, expected, name, compressed = false) {
|
||||
absolute(file, name); digest(expected, name);
|
||||
const bytes = await readFile(file); assert.equal(hash(bytes), expected, `${name} bytes changed`);
|
||||
return { bytes, value: JSON.parse((compressed ? gunzipSync(bytes, { maxOutputLength: 128 * 1024 * 1024 }) : bytes).toString('utf8')) };
|
||||
}
|
||||
|
||||
export async function validateVariantBuild(variant, descriptor) {
|
||||
assert.ok(descriptor && typeof descriptor === 'object', `Required ${variant} qualified build is missing`);
|
||||
absolute(descriptor.appRoot, `${variant} appRoot`); revision(descriptor.revision, variant);
|
||||
digest(descriptor.binarySha256, `${variant} binary`); revision(descriptor.rustSourceTree, `${variant} Rust source tree`);
|
||||
const receipt = (await hashedJson(descriptor.qualifiedReceiptFile, descriptor.qualifiedReceiptSha256, `${variant} qualified receipt`)).value;
|
||||
const summary = (await hashedJson(descriptor.inventorySummaryFile, descriptor.inventorySummarySha256, `${variant} inventory summary`)).value;
|
||||
const manifest = (await hashedJson(descriptor.manifestFile, descriptor.manifestFileSha256, `${variant} inventory manifest`, true)).value;
|
||||
digest(descriptor.aggregateSha256, `${variant} aggregate inventory`);
|
||||
const rows = receipt.results?.filter(row => row.variant === variant);
|
||||
assert.equal(rows?.length, 1, `${variant} qualified receipt must select this variant exactly once`);
|
||||
assert.equal(rows[0].revision, descriptor.revision, `${variant} qualified receipt revision mismatch`);
|
||||
assert.equal(rows[0].compiledBuildRevision, descriptor.revision, `${variant} qualified compiled revision mismatch`);
|
||||
assert.equal(rows[0].sourceClean, true, `${variant} qualified source was not clean`);
|
||||
const native = receipt.nativeStageHashes?.filter(row => row.variant === variant);
|
||||
assert.equal(native?.length, 1, `${variant} qualified native stage is missing`);
|
||||
assert.equal(native[0].binarySha256, descriptor.binarySha256, `${variant} native stage binary mismatch`);
|
||||
assert.equal(receipt.binarySha256, descriptor.binarySha256, `${variant} receipt binary mismatch`);
|
||||
assert.equal(receipt.rustSourceTree, descriptor.rustSourceTree, `${variant} Rust source identity mismatch`);
|
||||
assert.equal(receipt.rustInputObjects?.runner, descriptor.rustSourceTree, `${variant} qualified Rust input objects are missing`);
|
||||
absolute(native[0].targetRealPath, `${variant} qualified native target`);
|
||||
for (const [file, object] of Object.entries(receipt.rustInputObjects)) {
|
||||
assert.ok(file && !path.posix.isAbsolute(file) && path.posix.normalize(file) === file && !file.includes('\\') && !file.split('/').includes('..'), 'Qualified Rust input path is invalid');
|
||||
revision(object, `${variant} Rust input ${file}`);
|
||||
}
|
||||
assert.equal(receipt.artifactInventorySummary, descriptor.qualifiedInventorySummaryReference ?? descriptor.inventorySummaryFile, `${variant} receipt points to another inventory summary`);
|
||||
assert.equal(manifest.variant, variant); assert.equal(manifest.revision, descriptor.revision);
|
||||
assert.equal(manifest.compiledBuildRevision, descriptor.revision); assert.equal(manifest.aggregateSha256, descriptor.aggregateSha256);
|
||||
assert.deepEqual(manifest.roots, INVENTORY_ROOTS);
|
||||
assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount && manifest.fileCount > 0);
|
||||
const row = summary.inventories?.[variant]; assert.ok(row, `${variant} inventory summary entry missing`);
|
||||
assert.equal(row.manifest, path.basename(descriptor.manifestFile)); assert.equal(row.manifestSha256, descriptor.manifestFileSha256);
|
||||
assert.equal(row.aggregateSha256, descriptor.aggregateSha256); assert.equal(row.fileCount, manifest.fileCount);
|
||||
if (Object.hasOwn(BASELINES, variant)) {
|
||||
assert.equal(descriptor.revision, BASELINES[variant], 'Original baseline reference must remain unchanged');
|
||||
assert.equal(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'B0/B1 must use the original, unmodified aligned4a83 qualified receipt');
|
||||
assert.equal(descriptor.inventorySummarySha256, OLD_SUMMARY); assert.equal(descriptor.binarySha256, OLD_BINARY);
|
||||
assert.equal(descriptor.rustSourceTree, OLD_RUST);
|
||||
for (const [key, value] of Object.entries(OLD_INVENTORIES[variant])) assert.equal(descriptor[key], value, `Original ${variant} inventory changed`);
|
||||
} else {
|
||||
assert.equal(variant, 'B2');
|
||||
assert.ok(![...Object.values(BASELINES), OLD_B2].includes(descriptor.revision), 'Final B2 needs its new actual immutable product revision');
|
||||
assert.notEqual(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'Final B2 needs its own new qualified receipt');
|
||||
assert.notEqual(descriptor.binarySha256, OLD_BINARY, 'Final Rust B2 needs its newly qualified binary');
|
||||
assert.notEqual(descriptor.rustSourceTree, OLD_RUST, 'Final Rust B2 needs its newly qualified Rust inputs');
|
||||
assert.notEqual(native[0].targetRealPath, OLD_NATIVE_TARGET, 'Final Rust B2 must not reuse the original B0/B1 target');
|
||||
}
|
||||
return { receipt, summary, manifest };
|
||||
}
|
||||
|
||||
// Match the existing inventory generator and H5 helper exactly: Unicode code
|
||||
// point order, Python ensure_ascii JSON with sorted bytes/path/sha256 keys.
|
||||
function pythonOrder(left, right) {
|
||||
const a = [...left], b = [...right];
|
||||
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
||||
const difference = a[index].codePointAt(0) - b[index].codePointAt(0);
|
||||
if (difference) return difference;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })
|
||||
.replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
||||
function inventoryPath(value) {
|
||||
assert.ok(typeof value === 'string' && value.length && !value.includes('\\') && !value.includes('\0')
|
||||
&& !path.posix.isAbsolute(value) && path.posix.normalize(value) === value && !value.split('/').includes('..')
|
||||
&& INVENTORY_ROOTS.some(root => value.startsWith(`${root}/`)), 'Inventory filename escapes registered roots');
|
||||
return value;
|
||||
}
|
||||
export async function validateActualInventory(build, manifest) {
|
||||
const startedAtMs = Date.now(), started = performance.now(), canonicalRoot = await realpath(build.appRoot);
|
||||
assert.deepEqual(manifest.roots, INVENTORY_ROOTS);
|
||||
const expected = new Map(), canonicalFiles = new Set();
|
||||
for (const row of manifest.files) {
|
||||
const name = inventoryPath(row.path);
|
||||
assert.equal(expected.has(name), false, 'Duplicate inventory filename'); expected.set(name, row);
|
||||
digest(row.sha256, 'inventory file'); assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0);
|
||||
}
|
||||
assert.equal(expected.size, manifest.fileCount, 'Inventory file count is incomplete');
|
||||
const actualNames = [], canonicalDirectories = new Set();
|
||||
const walk = async relative => {
|
||||
const directory = path.join(build.appRoot, relative), canonical = await realpath(directory);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory directory resolves outside APP_ROOT');
|
||||
assert.equal(canonicalDirectories.has(canonical), false, 'Inventory directory aliases another registered directory'); canonicalDirectories.add(canonical);
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const child = `${relative}/${entry.name}`;
|
||||
if (entry.isDirectory()) await walk(child);
|
||||
else {
|
||||
const file = path.join(build.appRoot, child), metadata = await lstat(file);
|
||||
if (metadata.isSymbolicLink()) assert.equal((await stat(file)).isDirectory(), false, 'Inventory directory symlink is not a frozen file');
|
||||
assert.ok(metadata.isFile() || metadata.isSymbolicLink(), 'Inventory includes a nonregular entry');
|
||||
inventoryPath(child); actualNames.push(child);
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const root of INVENTORY_ROOTS) await walk(root);
|
||||
actualNames.sort(pythonOrder);
|
||||
assert.deepEqual(actualNames, [...expected.keys()].sort(pythonOrder), 'Actual APP_ROOT file set differs from frozen inventory');
|
||||
const actualRows = new Array(actualNames.length); let next = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(8, actualNames.length) }, async () => {
|
||||
for (;;) {
|
||||
const index = next++; if (index >= actualNames.length) return;
|
||||
const name = actualNames[index], file = path.join(build.appRoot, name), canonical = await realpath(file);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory file resolves outside APP_ROOT');
|
||||
assert.equal(canonicalFiles.has(canonical), false, 'Inventory filenames alias the same canonical file'); canonicalFiles.add(canonical);
|
||||
const before = await stat(file); assert.ok(before.isFile(), 'Inventory evidence must be a regular file');
|
||||
const content = createHash('sha256'); for await (const bytes of createReadStream(file)) content.update(bytes);
|
||||
const after = await stat(file);
|
||||
assert.equal(after.size, before.size, 'Artifact changed during inventory hashing'); assert.equal(after.mtimeMs, before.mtimeMs, 'Artifact changed during inventory hashing');
|
||||
const row = { bytes: after.size, path: name, sha256: content.digest('hex') }, frozen = expected.get(name);
|
||||
assert.equal(row.bytes, frozen.bytes, `Actual artifact size changed: ${name}`); assert.equal(row.sha256, frozen.sha256, `Actual artifact bytes changed: ${name}`);
|
||||
actualRows[index] = row;
|
||||
}
|
||||
}));
|
||||
const aggregateSha256 = hash(Buffer.from(`${actualRows.map(pythonRow).join('\n')}\n`, 'utf8'));
|
||||
assert.equal(aggregateSha256, manifest.aggregateSha256, 'Actual APP_ROOT aggregate differs from frozen inventory');
|
||||
assert.equal(aggregateSha256, build.aggregateSha256, 'Actual APP_ROOT aggregate differs from qualified build');
|
||||
return { appRoot: build.appRoot, canonicalAppRoot: canonicalRoot, fileCount: actualRows.length, aggregateSha256,
|
||||
startedAtMs, completedAtMs: Date.now(), elapsedMs: performance.now() - started,
|
||||
scope: 'Live registered file set, canonical containment/uniqueness, size and SHA checked before this arm; not an assertion of future immutability' };
|
||||
}
|
||||
|
||||
function harnessFields(release) {
|
||||
return Object.fromEntries(['harnessRevision', 'protocolSha256', 'comparatorSha256', 'inventoryHelperSha256', 'protocolDocumentSha256', 'lockSha256', 'gatesSha256', 'workloadSha256', 'observationMode', 'protocolVersion'].map(key => [key, release[key]]));
|
||||
}
|
||||
|
||||
export async function loadRelease(releaseFile, stage, outputRoot) {
|
||||
absolute(releaseFile, 'root release'); absolute(outputRoot, 'output root');
|
||||
const releaseBytes = await readFile(releaseFile), release = JSON.parse(releaseBytes);
|
||||
const plan = stagePlan(stage);
|
||||
assert.equal(release.authorized, true, 'Root release is not authorized'); assert.equal(release.stage, stage);
|
||||
assert.equal(release.outputRoot, outputRoot, 'Root release freezes a different exact output path');
|
||||
assert.equal(release.orchestratorSha256, hash(await readFile(selfFile)), 'Orchestrator changed after release');
|
||||
absolute(release.harnessRoot, 'H5 harness'); revision(release.harnessRevision, 'H5 harness');
|
||||
assert.equal(release.harnessRevision, H5_REVISION, 'Use the actual independently frozen H5 full revision');
|
||||
assert.equal(release.observationMode, MODE); assert.equal(release.protocolVersion, PROTOCOL);
|
||||
assert.equal(release.workloadSha256, WORKLOAD_SHA256);
|
||||
assert.equal(release.resourceWatchdogMillis, RESOURCE_WATCHDOG_MILLIS, 'Resource watchdog registration changed; this is not an SLA');
|
||||
assert.equal(release.lockSha256, LOCK_SHA256); assert.equal(release.gatesSha256, GATES_SHA256);
|
||||
for (const [key, expected] of [['protocolSha256', H5_PROTOCOL_SHA256], ['comparatorSha256', H5_COMPARATOR_SHA256], ['inventoryHelperSha256', H5_INVENTORY_HELPER_SHA256], ['protocolDocumentSha256', H5_PROTOCOL_DOCUMENT_SHA256]])
|
||||
assert.equal(release[key], expected, `Reviewed H5 source pin changed: ${key}`);
|
||||
for (const variant of plan.requiredVariants) await validateVariantBuild(variant, release.perVariantBuilds?.[variant]);
|
||||
const context = { releaseFile, releaseBytesSha256: hash(releaseBytes), release, plan, out: outputRoot, commands: [], commandIndex: 0 };
|
||||
await checkPriorStages(context);
|
||||
if (plan.noiseStage) await checkNoise(context);
|
||||
if (plan.finalizationOnly) await checkFinalizationInputs(context);
|
||||
return context;
|
||||
}
|
||||
|
||||
async function checkReleaseBytes(context) {
|
||||
assert.equal(hash(await readFile(context.releaseFile)), context.releaseBytesSha256, 'Exact root release bytes changed');
|
||||
assert.equal(hash(await readFile(selfFile)), context.release.orchestratorSha256, 'Orchestrator changed after release');
|
||||
if (context.rootReleaseCopy) assert.equal(hash(await readFile(context.rootReleaseCopy)), context.releaseBytesSha256, 'Preserved root release bytes changed');
|
||||
}
|
||||
|
||||
function checkComparisonIdentity(report, context) {
|
||||
assert.equal(report.complete, true); assert.equal(report.observationMode, MODE); assert.equal(report.protocolVersion, PROTOCOL);
|
||||
assert.equal(report.workloadSha256, WORKLOAD_SHA256); assert.equal(report.measurementSha256, context.release.protocolSha256);
|
||||
assert.equal(report.comparatorSha256, context.release.comparatorSha256); assert.equal(report.gatesSha256, GATES_SHA256);
|
||||
assert.equal(report.pairCount, 5);
|
||||
}
|
||||
|
||||
async function checkPriorStages(context) {
|
||||
for (const stage of context.plan.priorStages) {
|
||||
const pin = context.release.priorStageReports?.[stage]; assert.ok(pin, `Stage order requires frozen ${stage} first`);
|
||||
assert.equal(pin.file, path.join(context.out, `${stage}.json`), 'Prior stage report path changed');
|
||||
const report = (await hashedJson(pin.file, pin.sha256, stage)).value;
|
||||
checkComparisonIdentity(report, context);
|
||||
const priorPlan = stagePlan(stage);
|
||||
assert.equal(report.phase, priorPlan.aa ? 'aa' : 'compare');
|
||||
assert.equal(report.baselineRevision, BASELINES[priorPlan.baseline]);
|
||||
const candidateRevision = BASELINES[priorPlan.candidate] ?? context.release.perVariantBuilds.B2?.revision;
|
||||
assert.equal(report.candidateRevision, candidateRevision, 'Prior stage used a different measured candidate revision');
|
||||
for (const [arm, variant] of [['baseline', priorPlan.baseline], ['candidate', priorPlan.candidate]]) {
|
||||
const expected = context.release.perVariantBuilds?.[variant] ?? (BASELINES[variant] ? { ...OLD_INVENTORIES[variant], qualifiedReceiptSha256: OLD_RECEIPT, inventorySummarySha256: OLD_SUMMARY } : undefined);
|
||||
assert.ok(expected, 'Prior candidate qualified build is unavailable');
|
||||
for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256'])
|
||||
assert.equal(report.artifactInventories?.[arm]?.[key], expected[key], 'Prior stage changed a measured qualified inventory');
|
||||
}
|
||||
if (!priorPlan.aa) assert.equal(report.performancePassed, true, `Prior ${stage} performance failed`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkNoise(context) {
|
||||
const { release, plan, out } = context;
|
||||
const expected = path.join(out, `${plan.noiseStage}.json`);
|
||||
assert.equal(release.noiseReportFile, expected, 'Use the same exact frozen A/A noise path');
|
||||
const noise = (await hashedJson(expected, release.noiseReportSha256, 'Reviewed A/A noise report')).value;
|
||||
checkComparisonIdentity(noise, context); assert.equal(noise.phase, 'aa');
|
||||
assert.equal(noise.baselineRevision, release.perVariantBuilds[plan.baseline].revision);
|
||||
assert.equal(noise.candidateRevision, noise.baselineRevision);
|
||||
assert.equal(noise.h5Assessments?.length, 5, 'Reviewed A/A must contain all five complete assessments');
|
||||
assert.ok(noise.measurements?.length > 0 && noise.measurements.every(row => Number.isFinite(row.aaNoiseEnvelopeRelative) && row.aaNoiseEnvelopeRelative >= 0), 'A/A noise envelope is invalid');
|
||||
const build = release.perVariantBuilds[plan.baseline];
|
||||
for (const arm of ['baseline', 'candidate']) for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256'])
|
||||
assert.equal(noise.artifactInventories?.[arm]?.[key], build[key], 'Frozen A/A baseline inventory differs from required build');
|
||||
return noise;
|
||||
}
|
||||
|
||||
const QUIET_SETTLE_MILLIS = 30_000, QUIET_POLL_MILLIS = 1_000, QUIET_PS_TIMEOUT_MILLIS = 2_000;
|
||||
const QUIET_SCHEMA = 'paperclip.native-ingestion.physical-quiet.v1';
|
||||
const QUIET_SCOPE = 'Dedicated Linux process-name guard for specified Node/PostgreSQL/Runner/browser/Rust-build names only; not a claim that every possible host activity is absent. Qualified H5 Stop proof separately covers provider/sleep control groups.';
|
||||
const QUIET_ZOMBIE_POLICY = 'Strict: matched zombie/dead rows remain blockers until absent; no automatic killing or silent zombie exclusion.';
|
||||
function processFamily(comm) {
|
||||
if (/^(node|node-MainThread)$/.test(comm)) return 'node';
|
||||
if (/^(postgres|postmaster|pg_ctl|initdb)$/.test(comm)) return 'postgresql';
|
||||
if (/^(paperclip-runne|paperclip-runner.*|Runner)$/.test(comm)) return 'runner';
|
||||
if (/^(chrome|headless|chromium)/i.test(comm)) return 'browser';
|
||||
if (/^(rustc|cargo)$/.test(comm)) return 'rust-build';
|
||||
return null;
|
||||
}
|
||||
export function classifyLinuxProcesses(text, orchestratorPid = process.pid) {
|
||||
assert.ok(typeof text === 'string' && text.trim(), 'Linux ps returned no process rows');
|
||||
assert.ok(Number.isSafeInteger(orchestratorPid) && orchestratorPid > 0, 'Invalid orchestrator PID');
|
||||
const seen = new Set(), matched = []; let rowCount = 0;
|
||||
for (const line of text.trim().split(/\r?\n/)) {
|
||||
const fields = line.match(/^\s*(\d+)\s+(\d+)\s+(.+?)\s+([DIRSTtWXZPK])\s*$/);
|
||||
assert.ok(fields, 'Malformed Linux ps PID/PPID/comm/state row');
|
||||
const row = { pid: Number(fields[1]), ppid: Number(fields[2]), comm: fields[3].trim(), state: fields[4] };
|
||||
assert.ok(Number.isSafeInteger(row.pid) && row.pid > 0 && Number.isSafeInteger(row.ppid) && row.ppid >= 0 && row.comm.length > 0 && row.comm.length <= 256 && !row.comm.includes('\0'), 'Malformed Linux ps process fields');
|
||||
assert.equal(seen.has(row.pid), false, 'Duplicate Linux ps PID'); seen.add(row.pid); rowCount++;
|
||||
const family = processFamily(row.comm);
|
||||
if (family) matched.push({ ...row, family, allowed: row.pid === orchestratorPid && !['Z', 'X'].includes(row.state) });
|
||||
}
|
||||
assert.ok(seen.has(orchestratorPid), 'Linux ps omitted the current orchestrator PID');
|
||||
return { rowCount, matched, blockers: matched.filter(row => !row.allowed) };
|
||||
}
|
||||
function linuxProcessRows(timeoutMillis) {
|
||||
return execFileSync('/bin/ps', ['-eo', 'pid=,ppid=,comm=,state='], {
|
||||
encoding: 'utf8', timeout: timeoutMillis, maxBuffer: 2 * 1024 * 1024, env: { LC_ALL: 'C' },
|
||||
});
|
||||
}
|
||||
export async function physicalQuietGuard({ receiptFile, phase, armStage, primaryFailureAlreadyPresent = false,
|
||||
platform = process.platform, orchestratorPid = process.pid, readPs = linuxProcessRows,
|
||||
now = () => performance.now(), pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)) }) {
|
||||
absolute(receiptFile, 'physical quiet receipt');
|
||||
const started = now(), receipt = { schema: QUIET_SCHEMA, scope: QUIET_SCOPE, phase, armStage, platform, orchestratorPid,
|
||||
fields: ['pid', 'ppid', 'comm', 'state'], zombiePolicy: QUIET_ZOMBIE_POLICY,
|
||||
resourceSettleLimitMillis: QUIET_SETTLE_MILLIS, pollIntervalMillis: QUIET_POLL_MILLIS, psTimeoutMillis: QUIET_PS_TIMEOUT_MILLIS,
|
||||
primaryFailureAlreadyPresent, startedAtMs: Date.now(), passed: false, observations: [] };
|
||||
let failure;
|
||||
try {
|
||||
assert.equal(platform, 'linux', 'Accepted H5 physical quiet guard requires Linux');
|
||||
assert.ok(['pre-arm', 'post-arm'].includes(phase), 'Unknown physical quiet phase');
|
||||
for (;;) {
|
||||
const remaining = QUIET_SETTLE_MILLIS - (now() - started);
|
||||
assert.ok(remaining > 0, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
const observed = classifyLinuxProcesses(readPs(Math.max(1, Math.min(QUIET_PS_TIMEOUT_MILLIS, Math.floor(remaining)))), orchestratorPid);
|
||||
receipt.observations.push({ atMs: Date.now(), elapsedMillis: now() - started, ...observed });
|
||||
assert.ok(now() - started < QUIET_SETTLE_MILLIS, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
if (observed.blockers.length === 0) { receipt.passed = true; break; }
|
||||
const wait = Math.min(QUIET_POLL_MILLIS, QUIET_SETTLE_MILLIS - (now() - started));
|
||||
assert.ok(wait > 0, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
await pause(wait);
|
||||
}
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
receipt.failure = { name: error instanceof Error ? error.name : 'UnknownError', message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
receipt.completedAtMs = Date.now(); receipt.elapsedMillis = now() - started;
|
||||
const bytes = Buffer.from(JSON.stringify(receipt, null, 2) + '\n');
|
||||
await writeFile(receiptFile, bytes, { flag: 'wx' });
|
||||
const result = { file: receiptFile, bytes: bytes.length, sha256: hash(bytes), passed: receipt.passed };
|
||||
if (failure) {
|
||||
const error = new Error(`Physical quiet guard failed (${phase}); receipt=${receiptFile}; sha256=${result.sha256}`, { cause: failure });
|
||||
Object.assign(error, { receiptFile, receiptSha256: result.sha256 }); throw error;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
export async function assertNoPreferredUiShadow(appRoot, statPath = lstat) {
|
||||
const preferredIndex = path.join(appRoot, 'server/ui-dist/index.html'), preferredDirectory = path.dirname(preferredIndex);
|
||||
try { await statPath(preferredIndex); assert.fail('Preferred server/ui-dist/index.html exists (including a symlink); ui/dist inventory cannot attest the selected UI'); }
|
||||
catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
try { assert.equal((await statPath(preferredDirectory)).isSymbolicLink(), false, 'Preferred server/ui-dist directory symlink prevents the absence guard'); }
|
||||
catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
return { checkedAtMs: Date.now(), preferredIndex, preferredIndexAbsent: true, preferredDirectorySymlink: false,
|
||||
selectedUiRoot: path.join(appRoot, 'ui/dist'),
|
||||
assumption: 'Preferred server/ui-dist index absent at preflight; inventoried ui/dist is the compiled fallback. No routing/build changes or future immutability claim.' };
|
||||
}
|
||||
|
||||
async function checkFrozen(context, selectedVariant) {
|
||||
await checkReleaseBytes(context);
|
||||
for (const key of ['DATABASE_URL', 'PAPERCLIP_RUNNER_STATE_DIR', 'PAPERCLIP_INGESTION_SERVER_COMMAND', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR',
|
||||
'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR', 'PAPERCLIP_INGESTION_BROWSER_CAPTURE', 'NODE_OPTIONS'])
|
||||
assert.ok(!process.env[key], `Acceptance rejects ambient ${key}`);
|
||||
const { release, plan } = context, actualInventories = {};
|
||||
for (const variant of plan.requiredVariants) {
|
||||
const build = release.perVariantBuilds[variant], qualified = await validateVariantBuild(variant, build);
|
||||
assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.revision, `${variant} SUT revision changed`);
|
||||
assert.equal(execFileSync('git', ['status', '--short'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), '', `${variant} SUT is dirty`);
|
||||
assert.equal((await getJson(path.join(build.appRoot, 'server/dist/build-info.json'))).commit, build.revision);
|
||||
const binaryFile = path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd');
|
||||
const native = qualified.receipt.nativeStageHashes.find(row => row.variant === variant);
|
||||
const actualTarget = await realpath(path.join(build.appRoot, 'packages/paperclip-runner/runner/target'));
|
||||
assert.equal(actualTarget, native.targetRealPath, `${variant} actual native target differs from qualified receipt`);
|
||||
assert.equal(await realpath(binaryFile), path.join(actualTarget, 'release/paperclip-runnerd'), `${variant} native binary resolves outside its exact qualified target`);
|
||||
assert.equal(hash(await readFile(binaryFile)), build.binarySha256, `${variant} actual native binary changed`);
|
||||
for (const [file, object] of Object.entries(qualified.receipt.rustInputObjects))
|
||||
assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/${file}`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), object, `${variant} qualified Rust input changed: ${file}`);
|
||||
assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/runner`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.rustSourceTree);
|
||||
if (!selectedVariant || selectedVariant === variant) actualInventories[variant] = {
|
||||
...await validateActualInventory(build, qualified.manifest), uiServing: await assertNoPreferredUiShadow(build.appRoot),
|
||||
};
|
||||
}
|
||||
assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), release.harnessRevision);
|
||||
assert.equal(execFileSync('git', ['status', '--short'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), '');
|
||||
assert.equal(hash(Buffer.concat(await Promise.all(protocolFiles.map(file => readFile(path.join(release.harnessRoot, 'tests/e2e', file)))))), release.protocolSha256);
|
||||
for (const [file, expected] of [['scripts/compare-native-ingestion-browser.ts', release.comparatorSha256], ['scripts/native-ingestion-h5-inventory.mjs', release.inventoryHelperSha256], ['tests/e2e/fixtures/native-ingestion-h5.md', release.protocolDocumentSha256], ['scripts/native-ingestion-performance-gates.mjs', release.gatesSha256], ['pnpm-lock.yaml', release.lockSha256]])
|
||||
assert.equal(hash(await readFile(path.join(release.harnessRoot, file))), expected, `Frozen H5 support changed: ${file}`);
|
||||
await checkPriorStages(context);
|
||||
if (plan.noiseStage) await checkNoise(context);
|
||||
return { checkedAtMs: Date.now(), releaseFileSha256: context.releaseBytesSha256, actualInventories };
|
||||
}
|
||||
|
||||
async function execute(context, step) {
|
||||
const index = ++context.commandIndex, startedAt = new Date().toISOString(), log = path.join(context.out, `${step.stage}.log`);
|
||||
const command = { index, stage: step.stage, cwd: step.cwd, executable: process.execPath, args: step.args, environmentOverrides: step.env ?? {}, startedAt, log,
|
||||
resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS, resourceWatchdogScope: 'Resource-only cap including startup/test/teardown; original SLA and H5 observation limits remain driver-owned' };
|
||||
context.commands.push(command);
|
||||
const save = () => saveJson(path.join(context.out, `${context.plan.stage}-commands.json`), context.commands);
|
||||
await save(); console.log(`START ${step.stage} ${startedAt}; log=${log}`);
|
||||
const stream = (await open(log, 'wx')).createWriteStream();
|
||||
const child = spawn(process.execPath, step.args, { cwd: step.cwd, env: { ...process.env, ...step.env }, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
command.childPid = child.pid; await save();
|
||||
const interrupt = () => child.kill('SIGINT'), terminate = () => child.kill('SIGTERM');
|
||||
process.once('SIGINT', interrupt); process.once('SIGTERM', terminate);
|
||||
let forcedStop;
|
||||
const watchdog = setTimeout(() => {
|
||||
command.watchdogExpired = true; child.kill('SIGTERM');
|
||||
forcedStop = setTimeout(() => { command.forcedDriverKill = true; child.kill('SIGKILL'); }, 30_000);
|
||||
}, RESOURCE_WATCHDOG_MILLIS);
|
||||
child.stdout.pipe(stream, { end: false }); child.stderr.pipe(stream, { end: false });
|
||||
let exit;
|
||||
try { exit = await new Promise((resolve, reject) => { child.once('error', reject); child.once('close', (code, signal) => resolve({ code, signal })); }); }
|
||||
finally { clearTimeout(watchdog); clearTimeout(forcedStop); process.off('SIGINT', interrupt); process.off('SIGTERM', terminate); await new Promise(resolve => stream.end(resolve)); }
|
||||
Object.assign(command, { finishedAt: new Date().toISOString(), ...exit }); await save();
|
||||
console.log(`END ${step.stage} code=${exit.code} signal=${exit.signal ?? 'none'}`);
|
||||
assert.equal(command.watchdogExpired, undefined, 'Resource watchdog expired; stop for physical cleanup, never accept or advance');
|
||||
assert.equal(exit.code, 0, `Stage ${step.stage} failed; retain raw files and perform physical cleanup before continuing`);
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
|
||||
const port = server.address().port; await new Promise(resolve => server.close(resolve)); return port;
|
||||
}
|
||||
async function assertPortReleased(port) {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
}
|
||||
async function extractEvidence(directory) {
|
||||
const playwrightFile = path.join(directory, 'playwright.json'), report = await getJson(playwrightFile), attachments = [];
|
||||
const visit = node => { if (Array.isArray(node)) for (const child of node) visit(child); else if (node && typeof node === 'object') {
|
||||
if (Array.isArray(node.attachments)) attachments.push(...node.attachments.filter(value => value.name === 'native-ingestion-evidence'));
|
||||
for (const [key, child] of Object.entries(node)) if (key !== 'attachments') visit(child);
|
||||
} };
|
||||
visit(report); assert.equal(attachments.length, 1, 'Exactly one native-ingestion evidence attachment is required');
|
||||
const attachment = attachments[0];
|
||||
const evidence = attachment.body ? JSON.parse(Buffer.from(attachment.body, 'base64').toString()) : await getJson(path.resolve(directory, attachment.path));
|
||||
const file = path.join(directory, 'native-ingestion-evidence.json'); await saveJson(file, evidence, true);
|
||||
return { file, evidence, playwrightFile };
|
||||
}
|
||||
|
||||
async function armSidecar(directory, variant, build) {
|
||||
const destination = path.join(directory, 'inventory'); await mkdir(destination);
|
||||
const copies = {};
|
||||
for (const [key, sha] of [['manifestFile', 'manifestFileSha256'], ['inventorySummaryFile', 'inventorySummarySha256'], ['qualifiedReceiptFile', 'qualifiedReceiptSha256']]) {
|
||||
const bytes = await readFile(build[key]); assert.equal(hash(bytes), build[sha], 'Per-arm provenance changed while copying');
|
||||
const file = path.join(destination, path.basename(build[key])); await writeFile(file, bytes, { flag: 'wx' }); copies[key] = path.relative(directory, file);
|
||||
}
|
||||
const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant, sourceRevision: build.revision, compiledBuildRevision: build.revision,
|
||||
playwrightReportSha256: hash(await readFile(path.join(directory, 'playwright.json'))), ...copies,
|
||||
manifestFileSha256: build.manifestFileSha256, aggregateSha256: build.aggregateSha256,
|
||||
qualifiedReceiptSha256: build.qualifiedReceiptSha256, inventorySummarySha256: build.inventorySummarySha256 };
|
||||
const file = path.join(directory, 'arm-inventory.json'); await saveJson(file, sidecar, true); return file;
|
||||
}
|
||||
|
||||
async function armArtifactFiles(result) {
|
||||
const sidecar = await getJson(result.inventoryFile), base = path.dirname(result.inventoryFile);
|
||||
return [result.file, result.playwrightFile, result.inventoryFile,
|
||||
...['manifestFile', 'inventorySummaryFile', 'qualifiedReceiptFile'].map(key => path.resolve(base, sidecar[key])),
|
||||
path.join(base, 'pre-arm-inventory-check.json'), path.join(base, 'physical-http-port-check.json'),
|
||||
path.join(base, 'physical-quiet-pre.json'), path.join(base, 'physical-quiet-post.json')];
|
||||
}
|
||||
export async function sealArmArtifacts(out, result) {
|
||||
const canonicalRoot = await realpath(out), files = await armArtifactFiles(result), seen = new Set(), seals = [];
|
||||
for (const file of files) {
|
||||
absolute(file, 'raw arm artifact'); const canonical = await realpath(file);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Raw arm artifact resolves outside owned output');
|
||||
assert.equal(seen.has(canonical), false, 'Duplicate raw arm artifact'); seen.add(canonical);
|
||||
const bytes = await readFile(file); seals.push({ file, bytes: bytes.length, sha256: hash(bytes) });
|
||||
}
|
||||
return seals;
|
||||
}
|
||||
export async function verifyRawArtifactSeals(out, manifest) {
|
||||
const canonicalRoot = await realpath(out);
|
||||
for (const entry of manifest) for (const arm of ['baseline', 'candidate']) {
|
||||
const seals = entry[`${arm}Artifacts`]; assert.ok(Array.isArray(seals) && seals.length === 10, 'Raw arm byte seals are incomplete');
|
||||
const seen = new Set();
|
||||
for (const seal of seals) {
|
||||
absolute(seal.file, 'sealed arm artifact'); digest(seal.sha256, 'sealed arm artifact');
|
||||
assert.equal(seen.has(seal.file), false, 'Duplicate sealed arm artifact'); seen.add(seal.file);
|
||||
assert.ok((await realpath(seal.file)).startsWith(`${canonicalRoot}${path.sep}`), 'Sealed arm artifact resolves outside output');
|
||||
const bytes = await readFile(seal.file);
|
||||
assert.equal(bytes.length, seal.bytes, 'Raw arm artifact size changed after timing');
|
||||
assert.equal(hash(bytes), seal.sha256, 'Raw arm artifact bytes changed after timing');
|
||||
}
|
||||
const expected = await armArtifactFiles({ file: entry[`${arm}File`], playwrightFile: entry[`${arm}PlaywrightFile`], inventoryFile: entry[`${arm}InventoryFile`] });
|
||||
assert.deepEqual([...seen].sort(), expected.sort(), 'Raw seals do not cover this exact evidence/PW/sidecar/inventory set');
|
||||
for (const [filename, phase] of [['physical-quiet-pre.json', 'pre-arm'], ['physical-quiet-post.json', 'post-arm']]) {
|
||||
const receipt = await getJson(path.join(path.dirname(entry[`${arm}InventoryFile`]), filename));
|
||||
assert.equal(receipt.schema, QUIET_SCHEMA); assert.equal(receipt.passed, true); assert.equal(receipt.platform, 'linux');
|
||||
assert.equal(receipt.phase, phase); assert.equal(receipt.zombiePolicy, QUIET_ZOMBIE_POLICY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function physicalGuardReferences(manifest) {
|
||||
return manifest.flatMap((entry, pair) => ['baseline', 'candidate'].map(arm => ({ pair: pair + 1, arm,
|
||||
receipts: entry[`${arm}Artifacts`].filter(seal => /^physical-quiet-(pre|post)\.json$/.test(path.basename(seal.file))) })));
|
||||
}
|
||||
|
||||
async function browserRun(context, armStage, variant) {
|
||||
const preflight = await checkFrozen(context, variant); // ALL receipts; selected live artifact inventory before EVERY arm.
|
||||
const { release } = context, build = release.perVariantBuilds[variant], directory = path.join(context.out, armStage);
|
||||
await mkdir(directory, { recursive: false }); await saveJson(path.join(directory, 'pre-arm-inventory-check.json'), preflight, true);
|
||||
await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-pre.json'), phase: 'pre-arm', armStage });
|
||||
const port = await freePort();
|
||||
let primaryError, inventoryFile, extracted;
|
||||
try {
|
||||
await execute(context, { stage: armStage, cwd: release.harnessRoot,
|
||||
args: ['node_modules/@playwright/test/cli.js', 'test', '--config', 'tests/e2e/playwright-native-ingestion.config.ts'],
|
||||
env: { PAPERCLIP_INGESTION_OBSERVATION_MODE: MODE, PAPERCLIP_RUNNER_BINARY: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'),
|
||||
PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'),
|
||||
PAPERCLIP_INGESTION_TRANSPORT: 'sandbox-remote-with-local-stop', PAPERCLIP_E2E_PORT: String(port), PAPERCLIP_INGESTION_SMOKE: '0',
|
||||
PAPERCLIP_INGESTION_VARIANT: `${armStage}-${variant}`, PAPERCLIP_INGESTION_APP_ROOT: build.appRoot, PAPERCLIP_INGESTION_EVIDENCE_DIR: directory,
|
||||
PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES: 'Independent frozen H5 driver install; each measured variant owns its separately qualified application/native build' } });
|
||||
} catch (error) { primaryError = error; }
|
||||
try {
|
||||
inventoryFile = await armSidecar(directory, variant, build); extracted = await extractEvidence(directory);
|
||||
if (!primaryError) {
|
||||
const { evidence } = extracted;
|
||||
assert.equal(evidence.complete, true); assert.equal(evidence.observationComplete, true); assert.equal(evidence.observationMode, MODE);
|
||||
assert.equal(evidence.workloadSha256, WORKLOAD_SHA256); assert.equal(evidence.protocolVersion, PROTOCOL);
|
||||
assert.equal(evidence.transport, 'sandbox-remote-with-local-stop'); assert.equal(evidence.cleanup?.complete, true); assert.deepEqual(evidence.cleanup.errors, []);
|
||||
assert.equal(evidence.measurementSha256, release.protocolSha256); assert.equal(evidence.harnessProvenance?.sourceRevision, release.harnessRevision); assert.equal(evidence.harnessProvenance?.sourceStatus, '');
|
||||
assert.equal(evidence.sourceRevision, build.revision); assert.equal(evidence.compiledBuildRevision, build.revision);
|
||||
assert.equal(evidence.serverArtifact, 'compiled'); assert.equal(evidence.qualificationOnly, false); assert.equal(evidence.diagnosticOnly, false); assert.equal(evidence.performanceAcceptanceEligible, true);
|
||||
assert.equal(evidence.browserRecording?.trace, 'off'); assert.equal(evidence.browserRecording?.video, 'off'); assert.equal(evidence.browserRecording?.screencast, false);
|
||||
const loaded = evidence.cases.find(item => item.name === 'burst512'); assert.equal(loaded.runs, 100); assert.equal(loaded.participants, 100); assert.equal(loaded.durable.length, 100);
|
||||
const { validateArmInventory } = await import(pathToFileURL(path.join(release.harnessRoot, 'scripts/native-ingestion-h5-inventory.mjs')).href);
|
||||
await validateArmInventory(inventoryFile, { evidence, owningPlaywright: { reportSha256: hash(await readFile(extracted.playwrightFile)) } });
|
||||
}
|
||||
} catch (error) { primaryError ??= error; }
|
||||
try { await assertPortReleased(port); await saveJson(path.join(directory, 'physical-http-port-check.json'), { released: true, port, scope: 'Owned HTTP listener only; PG/Runner/provider physical cleanup is separately required before the next release' }, true); }
|
||||
catch (error) { primaryError ??= error; }
|
||||
try { await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-post.json'), phase: 'post-arm', armStage,
|
||||
primaryFailureAlreadyPresent: Boolean(primaryError) }); }
|
||||
catch (error) {
|
||||
if (primaryError) console.error(`Secondary physical quiet failure; primary fixture failure is retained: ${error.message}`);
|
||||
primaryError ??= error;
|
||||
} // Failed guard receipt and SHA remain independent evidence; the fixture failure stays primary.
|
||||
if (primaryError) throw primaryError;
|
||||
const result = { file: extracted.file, playwrightFile: extracted.playwrightFile, inventoryFile };
|
||||
return { ...result, artifacts: await sealArmArtifacts(context.out, result) };
|
||||
}
|
||||
|
||||
function comparatorArgs(context, plan, pairsFile, output, originalProof) {
|
||||
const args = ['scripts/compare-native-ingestion-browser.ts', `--observation-mode=${MODE}`, `--phase=${plan.aa ? 'aa' : 'compare'}`,
|
||||
`--expect=${plan.expectation}`, `--pairs=${pairsFile}`, `--output=${output}`];
|
||||
if (!plan.aa) args.push(`--candidate-policy=${plan.candidatePolicy}`, `--noise-report=${path.join(context.out, `${plan.noiseStage}.json`)}`,
|
||||
...(originalProof ? [`--original-h4d-proof=${originalProof.proofFile}`, `--original-h4d-inventory=${originalProof.inventoryFile}`, '--require-acceptance'] : ['--require-performance']));
|
||||
return args;
|
||||
}
|
||||
|
||||
async function browserSeries(context) {
|
||||
const { plan, release } = context, manifest = [], file = path.join(context.out, `${plan.stage}-pairs.json`), comparisonFile = path.join(context.out, `${plan.stage}.json`);
|
||||
await access(comparisonFile).then(() => assert.fail('Prior comparison output already exists'), error => { if (error.code !== 'ENOENT') throw error; });
|
||||
await saveJson(file, manifest, true);
|
||||
for (let pair = 0; pair < 5; pair++) {
|
||||
const entry = {}, arms = pair % 2 ? [['candidate', plan.candidate], ['baseline', plan.baseline]] : [['baseline', plan.baseline], ['candidate', plan.candidate]];
|
||||
for (const [arm, variant] of arms) {
|
||||
const result = await browserRun(context, `${plan.stage}-pair${pair + 1}-${arm}`, variant);
|
||||
entry[`${arm}File`] = result.file; entry[`${arm}PlaywrightFile`] = result.playwrightFile; entry[`${arm}InventoryFile`] = result.inventoryFile; entry[`${arm}Artifacts`] = result.artifacts;
|
||||
}
|
||||
manifest.push(entry); await saveJson(file, manifest);
|
||||
}
|
||||
await checkFrozen(context); // Same exact release/noise path and bytes pre-comparator.
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
const manifestSha256 = hash(await readFile(file));
|
||||
await execute(context, { stage: `${plan.stage}-comparison`, cwd: release.harnessRoot, args: comparatorArgs(context, plan, file, comparisonFile) });
|
||||
assert.equal(hash(await readFile(file)), manifestSha256, 'Raw paired manifest changed during comparison');
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
const result = await getJson(comparisonFile); checkComparisonIdentity(result, context);
|
||||
if (!plan.aa) assert.equal(result.performancePassed, true);
|
||||
await saveJson(path.join(context.out, `${plan.stage}.completion.json`), { stage: plan.stage, complete: true, fullArms: 10, completedAtMs: Date.now(),
|
||||
releaseFile: context.rootReleaseCopy, originalReleaseFile: context.releaseFile, releaseFileSha256: context.releaseBytesSha256, harnessPins: harnessFields(release),
|
||||
perVariantBuilds: Object.fromEntries(plan.requiredVariants.map(variant => [variant, release.perVariantBuilds[variant]])),
|
||||
manifestFile: file, manifestSha256, comparisonFile, comparisonSha256: hash(await readFile(comparisonFile)),
|
||||
physicalGuards: physicalGuardReferences(manifest),
|
||||
...(plan.noiseStage ? { noiseReportFile: release.noiseReportFile, noiseReportSha256: release.noiseReportSha256 } : {}),
|
||||
performancePassed: result.performancePassed ?? null, acceptancePassed: result.acceptancePassed ?? null,
|
||||
finalOriginalH4dConfirmationPending: plan.candidatePolicy === 'final-b2' }, true);
|
||||
}
|
||||
|
||||
async function checkFinalizationInputs(context) {
|
||||
const proof = context.release.originalH4dProof;
|
||||
assert.ok(proof, 'One later unchanged H4d proof is required for both final comparisons');
|
||||
for (const [key, sha] of [['proofFile', 'proofSha256'], ['inventoryFile', 'inventorySha256']]) await hashedJson(proof[key], proof[sha], `Original H4d ${key}`);
|
||||
const proofReport = await getJson(proof.proofFile), proofAttempts = [];
|
||||
const visit = value => {
|
||||
if (Array.isArray(value)) for (const child of value) visit(child);
|
||||
else if (value && typeof value === 'object') {
|
||||
if (Array.isArray(value.results) && Object.hasOwn(value, 'expectedStatus')) proofAttempts.push(...value.results);
|
||||
for (const [key, child] of Object.entries(value)) if (key !== 'results') visit(child);
|
||||
}
|
||||
};
|
||||
visit(proofReport); assert.equal(proofAttempts.length, 1, 'One later unchanged original H4d attempt must confirm both stages');
|
||||
const proofStartedAtMs = Date.parse(proofAttempts[0].startTime);
|
||||
assert.ok(Number.isFinite(proofStartedAtMs), 'Original H4d proof has no actual attempt start time');
|
||||
for (const stage of ['browser-b1-b2', 'browser-b0-b2']) {
|
||||
const pin = context.release.completedSeries?.[stage]; assert.ok(pin, `Missing frozen completed ${stage}`);
|
||||
assert.equal(pin.file, path.join(context.out, `${stage}.completion.json`));
|
||||
const completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value;
|
||||
assert.equal(completion.stage, stage); assert.equal(completion.complete, true); assert.equal(completion.fullArms, 10); assert.equal(completion.performancePassed, true);
|
||||
assert.ok(Number.isFinite(completion.completedAtMs) && completion.completedAtMs < proofStartedAtMs, 'Original H4d proof must run after BOTH timing stages complete');
|
||||
assert.deepEqual(completion.harnessPins, harnessFields(context.release));
|
||||
const plan = stagePlan(stage);
|
||||
for (const variant of plan.requiredVariants) assert.deepEqual(completion.perVariantBuilds[variant], context.release.perVariantBuilds[variant], 'Finalization changed a measured qualified build');
|
||||
assert.equal(completion.manifestFile, path.join(context.out, `${stage}-pairs.json`));
|
||||
assert.equal(completion.comparisonFile, path.join(context.out, `${stage}.json`));
|
||||
assert.equal(hash(await readFile(completion.manifestFile)), completion.manifestSha256, 'Raw timing manifest changed before finalization');
|
||||
const manifest = await getJson(completion.manifestFile); assert.equal(manifest.length, 5);
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
assert.deepEqual(completion.physicalGuards, physicalGuardReferences(manifest), 'Completion physical-guard receipts differ from sealed arm bytes');
|
||||
assert.equal(hash(await readFile(completion.comparisonFile)), completion.comparisonSha256, 'Preliminary performance assessment changed');
|
||||
assert.equal(completion.noiseReportFile, path.join(context.out, `${plan.noiseStage}.json`), 'Finalization rebound the frozen noise path');
|
||||
assert.equal(hash(await readFile(completion.noiseReportFile)), completion.noiseReportSha256, 'Frozen noise bytes changed before finalization');
|
||||
const originalRelease = await hashedJson(completion.releaseFile, completion.releaseFileSha256, `${stage} original release`);
|
||||
assert.equal(originalRelease.value.noiseReportFile, completion.noiseReportFile); assert.equal(originalRelease.value.noiseReportSha256, completion.noiseReportSha256);
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeSeries(context) {
|
||||
await checkFrozen(context); await checkFinalizationInputs(context);
|
||||
for (const stage of ['browser-b1-b2', 'browser-b0-b2']) {
|
||||
await checkFrozen(context); await checkFinalizationInputs(context);
|
||||
const pin = context.release.completedSeries[stage], completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value;
|
||||
const output = path.join(context.out, `${stage}.final.json`);
|
||||
await access(output).then(() => assert.fail('Final assessment output already exists'), error => { if (error.code !== 'ENOENT') throw error; });
|
||||
await execute(context, { stage: `${stage}-original-h4d-finalization`, cwd: context.release.harnessRoot,
|
||||
args: comparatorArgs(context, stagePlan(stage), completion.manifestFile, output, context.release.originalH4dProof) });
|
||||
await checkFinalizationInputs(context);
|
||||
assert.equal((await getJson(output)).acceptancePassed, true);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const stage = option('stage');
|
||||
if (!process.argv.includes('--execute') && !process.argv.includes('--validate-release')) {
|
||||
console.log(JSON.stringify({ draft: true, observationMode: MODE, protocolVersion: PROTOCOL, workloadSha256: WORKLOAD_SHA256,
|
||||
stages: STAGES.map(stagePlan), fullArms: 50, finalization: stagePlan(FINALIZE_STAGE),
|
||||
baselines: BASELINES, originalReceiptSha256: OLD_RECEIPT, resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS,
|
||||
frozenH5: { revision: H5_REVISION, protocolSha256: H5_PROTOCOL_SHA256, comparatorSha256: H5_COMPARATOR_SHA256, inventoryHelperSha256: H5_INVENTORY_HELPER_SHA256, protocolDocumentSha256: H5_PROTOCOL_DOCUMENT_SHA256 },
|
||||
pendingPins: 'Future B2 has no descriptor until its actual new source and separately qualified native/application build are available. No H5 VM execution is released by this file.',
|
||||
note: 'Plan only; no SUT/build/VM/network or arm starts. Root and independent peer review plus qualification release required.' }, null, 2)); return;
|
||||
}
|
||||
const out = option('output-root') ?? '/home/ubuntu/native-ingestion/evidence/h5/browser';
|
||||
const context = await loadRelease(option('release'), stage, out);
|
||||
if (process.argv.includes('--validate-release') && !process.argv.includes('--execute')) {
|
||||
console.log(JSON.stringify({ valid: true, stage, requiredVariants: context.plan.requiredVariants,
|
||||
validationScope: 'Static immutable release/receipt/manifest/noise/stage-order bindings only; no live SUT/driver/build attestation and no arm execution', releaseFileSha256: context.releaseBytesSha256 })); return;
|
||||
}
|
||||
await checkFrozen(context); await mkdir(out, { recursive: true });
|
||||
// Keep the exact release bytes as a distinct artifact, so later stages never
|
||||
// require an owner to retain an overwriteable release-input filename.
|
||||
context.rootReleaseCopy = path.join(out, `${stage}-root-release.json`);
|
||||
const releaseBytes = await readFile(context.releaseFile);
|
||||
assert.equal(hash(releaseBytes), context.releaseBytesSha256, 'Root release changed before preservation');
|
||||
await writeFile(context.rootReleaseCopy, releaseBytes, { flag: 'wx' });
|
||||
await saveJson(path.join(out, `${stage}-release-record.json`), { ...context.release, releaseFile: context.releaseFile,
|
||||
releaseFileSha256: context.releaseBytesSha256, recordedAt: new Date().toISOString() }, true);
|
||||
if (context.plan.finalizationOnly) await finalizeSeries(context); else await browserSeries(context);
|
||||
}
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === selfFile) await main();
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"schema": "paperclip.native-ingestion.h5b-helper-sources.v1",
|
||||
"scope": "Exact reviewed task-helper source snapshots. Per-run owner receipts remain authoritative for executed file hashes. This packaging did not execute a new driver, application, qualification or performance arm.",
|
||||
"frozenDriver": {
|
||||
"tag": "native-ingestion-h5b",
|
||||
"tagObject": "34498d7e09b61493e0f6703c354291c485836298",
|
||||
"commit": "179fe6fa6da1b24ca08c1162301538d3e3788af6",
|
||||
"lockSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8",
|
||||
"measurementSha256": "452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e",
|
||||
"comparatorSha256": "0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168",
|
||||
"inventoryHelperSha256": "76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9",
|
||||
"protocolDocumentSha256": "60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54"
|
||||
},
|
||||
"recordedLayout": {
|
||||
"base": "/home/ubuntu/native-ingestion",
|
||||
"driver": "harness-h5b",
|
||||
"helperDirectory": "helpers",
|
||||
"appDirectories": {
|
||||
"B0": "aligned-baseline",
|
||||
"B1": "aligned-cursors",
|
||||
"B2": "aligned-batching"
|
||||
}
|
||||
},
|
||||
"recordedAppRevisions": {
|
||||
"B0": "bac53ec3bfeaf6546e7aaeaee516487b30534654",
|
||||
"B1": "07061a601cdc20603c8c7a46f9ddbad7ac5f5939",
|
||||
"B2": "ca7cba171cd5198ed569d75012edc858041cbc44"
|
||||
},
|
||||
"rawB2Role": "Unoptimized batching base only. No future Rust candidate or matching build is supplied by this package.",
|
||||
"deliveryLockSha256AtPackaging": "7540075e96f9a93924a1e54775c17346c5760dd5d02902d83fb5228f918996ba",
|
||||
"deliveryLockWasExecutedDriver": false,
|
||||
"files": [
|
||||
{
|
||||
"file": "native-ingestion-run-h5b-browser.mjs",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-run-h5b-browser.mjs",
|
||||
"bytes": 53930,
|
||||
"sha256": "522d4fc718665ef484dec40ff56e463efe11b4ddf0c632c2d3f4811092d63fa6",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-linux-h5b-qualification.py",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-linux-h5b-qualification.py",
|
||||
"bytes": 6904,
|
||||
"sha256": "0d19a505ca860a7e8eb3975c4cef5a509b2ae449fab0d8d90dce830cb787219b",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-h5b-linux-readiness.mjs",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-h5b-linux-readiness.mjs",
|
||||
"bytes": 3446,
|
||||
"sha256": "5694d9ee274dfd0a1c77ac917c2bca84f6f0a858f4eabdd73e03241e02eda62e",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-extract-h5b-qualification.py",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-extract-h5b-qualification.py",
|
||||
"bytes": 5704,
|
||||
"sha256": "a052375a80b3abad4513cdf019325e83fed63c67d935f1dc2accba753ce589f0",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": [],
|
||||
"reviewedSourceRevision": "Root-approved post-read strengthening; no measured driver/source/workload/deadline change",
|
||||
"historicalSmallValidator": {
|
||||
"sha256": "00930574798e3d56257a4913d82400ccd739b4bb52e9a24d5cb18a36892be970",
|
||||
"bytes": 5445,
|
||||
"usedBy": "Recorded SMALL independent post-read proof; source retained in the owning raw23-fileSMALLartifactmanifest",
|
||||
"difference": "Packaged validator adds3existing-historypreludeProof assertions: run identity,1000losslessunique deltas, pre-epochverification/retainedhead. This packagedvalidator did not produce the SMALL result."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
# H5b independent-agent observation registration (prospective; no execution release)
|
||||
|
||||
H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5`
|
||||
(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`).
|
||||
Its observed 120-second streaming failures remain failures. Aborted or profiled
|
||||
attempts cannot be reused as complete timing arms. H5b uses one normal agent per concurrent remote participant. It repairs the
|
||||
experiment topology while preserving observation of original deadline outcomes.
|
||||
It does not optimize the product or turn earlier failures into passes.
|
||||
|
||||
The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654`
|
||||
and B1 `07061a601cdc20603c8c7a46f9ddbad7ac5f5939`. Raw B2
|
||||
`ca7cba171cd5198ed569d75012edc858041cbc44` is the unoptimized batching base,
|
||||
not a qualified future performance candidate. A later candidate needs its actual
|
||||
new immutable revision, normal build receipt and inventory; no candidate reference
|
||||
or matching binary is presumed before those artifacts exist. The historical
|
||||
a365/d8cd/914 application failures remain evidence of their recorded revisions.
|
||||
|
||||
Select H5 explicitly with
|
||||
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5b-observe-original-sla-independent-agents`.
|
||||
The new default H4d mode uses the same independent-agent geometry with the original
|
||||
120/180/900-second deadlines. Observation mode uses the explicit H5b protocol.
|
||||
Both modes hash the same nine source files, including the shared topology helper.
|
||||
Their source revision and measurement bytes must match for later confirmation.
|
||||
Old H4c/H5a protocols, geometry, arms and noise are ineligible for H5b.
|
||||
|
||||
Run `node scripts/describe-native-ingestion-h5.mjs` to inspect the registration.
|
||||
This is a dry run only: it starts no app, provider, database or browser and
|
||||
accepts no execution arguments. It grants no runtime release.
|
||||
|
||||
## Independent participant geometry and setup
|
||||
|
||||
Create a pool of100 normal remote agents for the full fixture (three for smoke).
|
||||
Each concurrent remote participant receives a distinct pool agent by index; the
|
||||
pool is reused only across sequential cases. The100-run case is100 agents × one
|
||||
run each. Agent POST, normal environment selection, managed files, permission
|
||||
checks and maxConcurrentRuns50 configuration stay enabled. The separate local
|
||||
Stop control agent and its normal local environment stay unchanged.
|
||||
|
||||
Use exactly eight setup workers, each awaiting normal agent creation and the
|
||||
existing60s lifecycle-ready poll. Keep index order despite completion order.
|
||||
Attempt and settle the whole pool; any setup error fails the invocation before
|
||||
case activation. Record every attempt, start/settlement time and status. No
|
||||
partial pool, silent retry or reduced load can qualify. The existing per-case
|
||||
READ ONLY authority snapshot must prove distinct agent_id values for every
|
||||
remote participant, all drawn from the ready pool. No event polling is added.
|
||||
|
||||
Agent creation occurs before the existing300s activation/readiness window starts.
|
||||
That start point remains after the pool and background issue creation. All agent
|
||||
setup remains inside the owning public900s default/1200s observation invocation;
|
||||
no setup work is moved outside those timers or into measured epochs. The existing
|
||||
waves of two provider activations, readiness polls and common source epoch stay
|
||||
unchanged. All arms use the same geometry and eight-worker setup. Its actual
|
||||
setup cost and100-agent full qualification are unverified until measured.
|
||||
|
||||
This changes managed-directory contention and agent-list cardinality. A result
|
||||
will describe independent-agent ingestion, not50 concurrent runs sharing one
|
||||
agent. Do not generalize it to the retained old2×50 failure below.
|
||||
|
||||
## Retained H5a negative result
|
||||
|
||||
H5a7388 on application B0a365 used two normal agents with50 concurrent runs each.
|
||||
All100 participants were admitted and clock-ready. The original120s poll failed
|
||||
with54/512 visible markers; all512 became visible296.555s after that wait began.
|
||||
All100 runs reached a terminal status55.234s later, with86 succeeded and14 failed.
|
||||
Independent read-only PostgreSQL checks found all51,200 expected lossless deltas
|
||||
and one accepted result for each run. This did not pass full qualification or A/A.
|
||||
|
||||
Each of the14 failed runs recorded native_session_interrupted from a30s
|
||||
agent-directory merge-lock timeout; each finalization retained retryable_failure
|
||||
with native_workspace_sync_out_failed and an ACTIVE unreleased lease. Seven
|
||||
failures mapped to each agent's shared file-sync root. Task workspace and lease
|
||||
identities were distinct; no provider failure was recorded. The new geometry
|
||||
removes that shared-agent contention from this experiment. It does not fix or
|
||||
qualify the product's old2×50 concurrency limit or justify disabling file sync.
|
||||
|
||||
A separate background presentation40P01 involved a run that succeeded. The
|
||||
retained log shows its coordinator→issue→run wait, but lacks PostgreSQL's full
|
||||
opposing lock cycle. Its cause and relationship to other contention are unproved.
|
||||
It is not evidence that the14 run failures came from a database deadlock. All
|
||||
historical failures remain separate; no failed H4c/H5a arm is reused as H5b noise.
|
||||
|
||||
## Fixed work and observation boundaries
|
||||
|
||||
Keep the five streaming event workloads and original local Stop case exactly as H4c:
|
||||
paced 1×128 at25ms; burst 1×16; burst10×128; burst100×512; existing-history
|
||||
1000 foreground prelude deltas plus measured10×128; local Stop1×512 at100ms,
|
||||
clicked after at least three observed markers. Keep normal API/UI activation,
|
||||
waves of two, shared300s readiness, actual GET+clock hellos, verified TLS,
|
||||
closed READ ONLY REPEATABLE READ authority snapshot, source identities and
|
||||
payloads, original nonawaited source-clock writes, and the existing2RAF observer.
|
||||
No normalized-event polling channel, per-event HTTP, injected provider receipt,
|
||||
raw recovery-policy change, diagnostic instrumentation or replacement app entry
|
||||
is added. Fake-plugin streaming remains separate from real-cloud Stop proof.
|
||||
|
||||
| Boundary | Original requirement | H5 observation |
|
||||
| --- | --- | --- |
|
||||
| Foreground markers | Original Playwright120s wait | Continue to an absolute600s ceiling from that same wait start |
|
||||
| All terminal runs |180s after foreground completion (and Stop click) | Same180s after eventual foreground completion; hard failure at the ceiling |
|
||||
| Stop acknowledgement and leases | Existing30s polls and ordering | Unchanged; full click→acknowledgement metric retained |
|
||||
| Invocation | Original900s test budget | Actual1200s observation timeout, separately disclosed |
|
||||
|
||||
The first120s poll is the original matcher. Only that exact known matcher
|
||||
deadline can extend observation. Page/API errors, other assertions, terminal
|
||||
failures and cleanup failures never become an allowed SLA miss. There is no
|
||||
collection wait when markers are available. The600s ceiling is absolute from
|
||||
the original wait start; continued observation uses only its remaining budget.
|
||||
|
||||
Record the original wait start/deadline, actual last count and its observation
|
||||
time, assessment time and error before continuing. Keep the original failure
|
||||
after completion. The original120s clock starts before the common source epoch;
|
||||
it is not silently rebased to epoch+120s. Derive a separate paint-deadline count
|
||||
from retained2RAF timestamps; do not mislabel it as the original poll's count.
|
||||
|
||||
The recorded `deadlineMs` is the nominal configured boundary and
|
||||
`assessedAtMs` is the actual matcher return time. Frozen Playwright1.62.1 can
|
||||
report its timeout before that nominal boundary when the next polling interval
|
||||
would cross it. Its original matcher outcome, known timeout error and incomplete
|
||||
count establish the original failure; timestamp ordering against the nominal
|
||||
deadline does not change that outcome. Retain both raw times and continue only
|
||||
until the unchanged absolute600s observation ceiling.
|
||||
|
||||
If120s fails, original terminal processing was **not reached**. Later180s
|
||||
terminal observations are separately labeled and cannot make the original
|
||||
protocol pass. Keep epoch→all-terminal drainage unshortened. An additional
|
||||
independent completion upper bound follows existing screenshot, reload,
|
||||
persistence, clock and lease checks; it includes verification overhead and is
|
||||
not an exact last-commit time.
|
||||
|
||||
The owning public Playwright JSON result supplies duration, startTime, retry,
|
||||
status, errors and configured timeout. Its observed total duration is an
|
||||
independent conservative assessment of the original900s budget; hooks/teardown
|
||||
can be treated differently by the original timeout. It is not an exact original
|
||||
timer-expiration observation. A final candidate must have total duration<900s
|
||||
in every arm and later pass one unchanged H4d default-mode full invocation on the
|
||||
same final SUT revision and full qualified artifact inventory.
|
||||
|
||||
## Complete evidence and eligibility
|
||||
|
||||
H5 `observationComplete` means all six cases reached finite correct outcomes,
|
||||
with every expected participant and foreground marker. It does not establish a
|
||||
pass of the original deadlines. Every remote run must succeed in native mode, retain every normalized
|
||||
delta's complete payload/identity/order, have exactly one independently read
|
||||
`run.result.accepted`, and pass the existing reload/result/clock/lease checks.
|
||||
The100-run case must independently verify all51200 deltas. Stop keeps its
|
||||
original cancellation, durable acknowledgement, local journal, PID/group/provider
|
||||
absence and lease checks; it must not acquire an accepted success result.
|
||||
|
||||
An observation ceiling, failed/cancelled remote run, missing event/result,
|
||||
unexpected API error, incomplete source clock, replay conflict or cleanup error
|
||||
is ineligible. Preserve its raw partial evidence and censoring state. Never
|
||||
substitute the ceiling as latency, compute a512-event tail from a visible prefix,
|
||||
discard a slow arm, retry it into the sample, cancel background work to make a
|
||||
pair finish, or hide work remaining after the measured interval.
|
||||
|
||||
Each comparison-manifest entry contains `baselineFile`, `candidateFile`,
|
||||
`baselinePlaywrightFile`, `candidatePlaywrightFile`, `baselineInventoryFile`, and
|
||||
`candidateInventoryFile`. The owning Playwright result must contain exactly that
|
||||
evidence attachment, one passed observation attempt, retry0, no errors, and the
|
||||
actual1200s timeout. An inventory sidecar binds that report hash to the exact
|
||||
gzip inventory bytes, full descriptor aggregate, summary and qualified build
|
||||
receipt hashes, source/build revision, and reported serving/native fingerprints.
|
||||
These frozen artifacts are evidence of the qualified build; the external owner
|
||||
still checks the live checkout/build inventories before each arm. A sidecar does
|
||||
not itself attest that the live filesystem stayed unchanged.
|
||||
|
||||
| Role | Allowed original failures | Required result |
|
||||
| --- | --- | --- |
|
||||
| Fresh B0/B1 A/A arms | Known burst512 streaming120 miss; explicit conservative observed-duration900 miss | Complete finite correct observation; root-reviewed noise suitability |
|
||||
| B0→B1 cursor-only candidate | Only an exact registered known miss also present in the paired baseline: burst512 streaming120 or conservative observed-duration900 | No additional deadline failures; latency/control nonregression plus separate incremental SQL evidence; never claim SLA fixed |
|
||||
| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later H4d proof |
|
||||
|
||||
Only burst512's streaming miss is registered as a known workload SLA failure.
|
||||
New failures in other workloads require diagnosis. The existing Stop poll
|
||||
ordering is preserved; no invented absolute click→ack30s requirement replaces it.
|
||||
|
||||
## Stage order and acceptance
|
||||
|
||||
Prospectively freeze **B0 A/A → B1 A/A → B0→B1 nonregression → B1→B2 → B0→B2**.
|
||||
Each stage has exactly five fresh alternating pairs, separate disposable state
|
||||
per arm, matched payloads/load/resource/cache policy, no reused baseline arms,
|
||||
and no concurrent builds/profilers/benchmarks on the measurement VM. Root reviews
|
||||
and freezes both complete A/A envelopes before any further product fix. Local
|
||||
product implementation can then proceed while the unchanged B0→B1 stage runs;
|
||||
the candidate VM build waits until that stage is quiet.
|
||||
|
||||
Keep the existing per-metric maximum absolute paired A/A envelope, median gate,
|
||||
and at least4/5 individual noise-exceeding loaded wins. Each loaded p95
|
||||
emission-to-visible, p95 intended-to-visible and terminal-drainage metric must
|
||||
improve; other fixed latency/control metrics must not regress outside noise.
|
||||
Do not infer p99 from sparse observations. There is no new fixed percentage A/A
|
||||
cutoff: root must reject unsuitable noisy/nonstationary calibration explicitly.
|
||||
|
||||
Use `--observation-mode=h5b-observe-original-sla-independent-agents` for the comparator. Candidate
|
||||
comparisons also require `--candidate-policy=cursor-nonregression --expect=nonregression`
|
||||
or `--candidate-policy=final-b2`. Final comparisons expose `performancePassed`
|
||||
separately; `acceptancePassed` remains false until `--original-h4d-proof` and
|
||||
`--original-h4d-inventory` establish the later fresh H4d default-deadline functional pass.
|
||||
That H4d proof is full, unprofiled, retry0, default mode with original120/180/900s
|
||||
bounds, on the same frozen H5b harness source, nine-file measurement bytes and
|
||||
independent-agent geometry, and the exact final SUT revision/qualified inventory.
|
||||
It uses a new disposable company and starts after all candidate arms. Old7d/H4c
|
||||
proofs and H5a observations cannot confirm this geometry. Prior failed H4c attempts remain in the record.
|
||||
Confirmation reuses the measured arms' authority, physical Stop, local lease and
|
||||
remote lease binding/settlement checks. The original30s polls keep their existing
|
||||
ordering; no absolute click-to-acknowledgement deadline is added.
|
||||
Use the explicit H5 `--require-performance` gate for the two final comparison
|
||||
stages while that later proof is pending; this requires every performance and
|
||||
candidate original-SLA gate, but grants no final acceptance. After the one later
|
||||
H4d default-mode qualification, recompute both reports against their unchanged paired
|
||||
manifests with the proof and `--require-acceptance`. This re-evaluates retained
|
||||
evidence; it neither reuses an arm in a different comparison nor reruns failures.
|
||||
|
||||
Before any runtime release, review the exact H5 protocol/comparator/helper
|
||||
hashes, registration dry run, outcome/eligibility tests, types and test listing.
|
||||
Then qualify the changed observation path before fresh A/A. Each later stage
|
||||
requires a separate root release pinning exact driver, build/inventory and
|
||||
reviewed noise-report bytes; verify the noise hash before the first arm and
|
||||
again before comparison. Product or workload changes require new prospective
|
||||
provenance and affected requalification; do not loosen gates after results.
|
||||
|
||||
Separate successful full-window diagnostics may later assess Node/PG CPU,
|
||||
queue/backlog, delay and memory with identical capture hooks. Aborted profiles
|
||||
remain diagnostic-only and cannot establish complete CPU/event costs. No CPU
|
||||
requirement or profiling is added to this timing comparator. Local results do
|
||||
not establish deployed improvement.
|
||||
@@ -0,0 +1,254 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const H5_MODE = 'h5b-observe-original-sla-independent-agents';
|
||||
export const H5_PROTOCOL = 'v5b-independent-agents-complete-observation-original-sla';
|
||||
export const H4D_PROTOCOL = 'v4d-independent-agents-original-deadlines';
|
||||
export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant',
|
||||
fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50,
|
||||
poolReuse: 'only-across-sequential-cases' });
|
||||
export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654',
|
||||
B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' });
|
||||
export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000,
|
||||
streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 });
|
||||
export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });
|
||||
export const H5_WORKLOADS = Object.freeze([
|
||||
{ name: 'paced', runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: 'burst16', runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'burst128', runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'burst512', runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false },
|
||||
{ name: 'existing-history', runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false },
|
||||
{ name: 'operator-stop', runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
|
||||
]);
|
||||
export const H5_REGISTRATION = Object.freeze({ mode: H5_MODE, protocolVersion: H5_PROTOCOL,
|
||||
baselineRevisions: H5_BASELINE_REVISIONS,
|
||||
topology: H5_TOPOLOGY, originalDeadlineProtocol: H4D_PROTOCOL,
|
||||
originalBudgets: ORIGINAL_BUDGETS, observationLimits: H5_LIMITS, workloads: H5_WORKLOADS,
|
||||
pairs: 5, order: 'baseline-first on even pair index; candidate-first on odd pair index',
|
||||
stages: ['B0 A/A', 'B1 A/A', 'B0→B1 nonregression', 'B1→B2 improvement', 'B0→B2 improvement'],
|
||||
knownBaselineMisses: ['burst512:streaming120', 'invocation:observed-total-duration900'],
|
||||
baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of the H4d original deadlines',
|
||||
cursorCandidateEligibility: 'Nonregression; only the paired baseline exact registered known misses may be inherited',
|
||||
finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later H4d default-mode functional PASS on the same harness source and independent-agent geometry required',
|
||||
noise: 'Fresh B0 and B1 A/A frozen before any further product implementation; no old abort/profile samples',
|
||||
performance: 'Loaded p95 emission/intended-to-visible and terminal drainage improve beyond frozen noise in median and at least4/5 pairs; all other metrics nonregress',
|
||||
});
|
||||
export const H5_WORKLOAD_SHA256 = createHash('sha256').update(JSON.stringify(H5_REGISTRATION)).digest('hex');
|
||||
|
||||
/** Agent setup settles in full before any case activation; its time remains
|
||||
* inside the owning invocation and outside the existing admission300s window. */
|
||||
export async function prepareParticipantAgents({ count, createAndReady, now = Date.now, onOutcome = _value => {} }) {
|
||||
assert.ok(Number.isSafeInteger(count) && count > 0 && count <= H5_TOPOLOGY.fullRemoteAgentPoolSize);
|
||||
const agents = new Array(count), failures = [];
|
||||
const outcome = { count, concurrency: H5_TOPOLOGY.agentSetupConcurrency, startedAtMs: now(), settledAtMs: null,
|
||||
complete: false, attempts: [] };
|
||||
onOutcome(outcome);
|
||||
let next = 0;
|
||||
const workers = await Promise.allSettled(Array.from({ length: Math.min(count, outcome.concurrency) }, async () => {
|
||||
for (;;) {
|
||||
const index = next++; if (index >= count) return;
|
||||
const attempt = { index, startedAtMs: now(), settledAtMs: null, status: 'pending', agentId: null };
|
||||
outcome.attempts.push(attempt); onOutcome(outcome);
|
||||
try {
|
||||
agents[index] = await createAndReady(index);
|
||||
assert.ok(typeof agents[index]?.id === 'string', 'Missing ready participant agent');
|
||||
attempt.agentId = agents[index].id; attempt.status = 'fulfilled';
|
||||
} catch (error) {
|
||||
failures.push(error); attempt.status = 'rejected'; attempt.error = String(error);
|
||||
} finally { attempt.settledAtMs = now(); onOutcome(outcome); }
|
||||
}
|
||||
}));
|
||||
for (const worker of workers) if (worker.status === 'rejected') failures.push(worker.reason);
|
||||
if (!failures.length) try { assert.equal(new Set(agents.map(agent => agent.id)).size, count, 'Participant agents must be distinct'); }
|
||||
catch (error) { failures.push(error); }
|
||||
outcome.settledAtMs = now(); outcome.complete = failures.length === 0; onOutcome(outcome);
|
||||
if (failures.length) throw new AggregateError(failures, 'Participant agent setup failed; no case may activate');
|
||||
return agents;
|
||||
}
|
||||
|
||||
export function assertParticipantSetup(report) {
|
||||
assert.deepEqual(report.agentTopology, { ...H5_TOPOLOGY, remoteAgentPoolSize: report.qualificationOnly ? 3 : 100 });
|
||||
const ids = report.remote?.agentIds;
|
||||
assert.equal(ids?.length, report.agentTopology.remoteAgentPoolSize, 'Incomplete independent participant agent pool');
|
||||
assert.equal(new Set(ids).size, ids.length, 'Duplicate participant agent pool identity');
|
||||
const setup = report.remote.agentSetup;
|
||||
assert.equal(setup?.complete, true, 'Agent setup must settle successfully before any case');
|
||||
assert.equal(setup.count, ids.length); assert.equal(setup.concurrency, H5_TOPOLOGY.agentSetupConcurrency);
|
||||
assert.ok(Number.isSafeInteger(report.bodyStartedAtMs) && report.bodyStartedAtMs > 0);
|
||||
assert.ok(Number.isSafeInteger(setup.startedAtMs) && setup.startedAtMs >= report.bodyStartedAtMs);
|
||||
assert.ok(Number.isSafeInteger(setup.settledAtMs) && setup.settledAtMs >= setup.startedAtMs);
|
||||
assert.equal(setup.attempts?.length, ids.length);
|
||||
assert.deepEqual(setup.attempts.map(attempt => attempt.index).sort((a, b) => a - b), ids.map((_id, index) => index));
|
||||
const boundaries = [];
|
||||
for (const attempt of setup.attempts) {
|
||||
assert.equal(attempt.status, 'fulfilled', 'Rejected agent setup cannot qualify');
|
||||
assert.equal(attempt.agentId, ids[attempt.index]);
|
||||
assert.ok(Number.isSafeInteger(attempt.startedAtMs) && attempt.startedAtMs >= setup.startedAtMs);
|
||||
assert.ok(Number.isSafeInteger(attempt.settledAtMs) && attempt.settledAtMs >= attempt.startedAtMs && attempt.settledAtMs <= setup.settledAtMs);
|
||||
if (attempt.settledAtMs > attempt.startedAtMs) boundaries.push([attempt.startedAtMs, 1], [attempt.settledAtMs, -1]);
|
||||
}
|
||||
let active = 0;
|
||||
for (const [_time, delta] of boundaries.sort((a, b) => a[0] - b[0] || a[1] - b[1])) {
|
||||
active += delta; assert.ok(active <= setup.concurrency, 'Agent setup exceeded the fixed concurrency');
|
||||
}
|
||||
for (const item of report.cases) assert.ok(item.activations?.[0]?.startedAtMs >= setup.settledAtMs,
|
||||
'Agent setup must finish before every case activation');
|
||||
}
|
||||
|
||||
export function assertParticipantTopology(report, result) {
|
||||
const ids = report.remote?.agentIds;
|
||||
if (!result.stop) {
|
||||
const rows = result.controllerAuthority?.rows;
|
||||
assert.equal(rows?.length, result.runs, 'Missing participant authority rows');
|
||||
assert.equal(new Set(rows.map(row => row.agent_id)).size, result.runs, 'Concurrent participants must use distinct normal agents');
|
||||
assert.ok(rows.every(row => ids.includes(row.agent_id)), 'Participant authority is outside the normal agent pool');
|
||||
}
|
||||
}
|
||||
|
||||
export function h5Enabled(value) {
|
||||
assert.ok(value === undefined || value === H5_MODE, 'Explicit supported H5 observation mode required');
|
||||
return value === H5_MODE;
|
||||
}
|
||||
|
||||
// Keep the original Playwright poll as the first wait. Only its known matcher
|
||||
// deadline may extend observation; callback errors and other failures propagate.
|
||||
export async function observeStreaming({ readCount, poll, target, now = Date.now, onOutcome = _value => {} }) {
|
||||
const startedAtMs = now();
|
||||
let lastObservedCount = null, lastObservedAtMs = null, readError;
|
||||
const read = async () => {
|
||||
try {
|
||||
const count = await readCount();
|
||||
assert.ok(Number.isSafeInteger(count) && count >= 0, 'Invalid visible marker count');
|
||||
lastObservedCount = count; lastObservedAtMs = now(); return count;
|
||||
} catch (error) { readError = error; throw error; }
|
||||
};
|
||||
let deadlineError;
|
||||
try { await poll(read, ORIGINAL_BUDGETS.streamingMs, target); }
|
||||
catch (error) {
|
||||
if (readError) throw readError;
|
||||
assert.ok(error?.matcherResult?.name === 'toBeGreaterThanOrEqual' && error.matcherResult.pass === false
|
||||
&& error.message.includes(`Timeout ${ORIGINAL_BUDGETS.streamingMs}ms exceeded while waiting on the predicate`),
|
||||
'Only the original streaming matcher deadline can extend observation');
|
||||
assert.ok(lastObservedCount !== null && lastObservedCount < target, 'Deadline needs an incomplete actual observation');
|
||||
deadlineError = error;
|
||||
}
|
||||
if (readError) throw readError;
|
||||
// deadlineMs is the nominal configured boundary. Playwright can report its
|
||||
// timeout earlier when its next polling interval would cross that boundary.
|
||||
const original = { status: deadlineError ? 'failed' : 'passed', reason: deadlineError ? 'original_streaming_deadline' : null,
|
||||
startedAtMs, deadlineMs: startedAtMs + ORIGINAL_BUDGETS.streamingMs, assessedAtMs: now(),
|
||||
target, lastObservedCount, lastObservedAtMs, error: deadlineError?.message ?? null };
|
||||
const observation = { ceilingMs: H5_LIMITS.streamingMs, deadlineMs: startedAtMs + H5_LIMITS.streamingMs,
|
||||
complete: false, censored: false, completedAtMs: null };
|
||||
const result = { original, observation };
|
||||
onOutcome(result);
|
||||
if (deadlineError) {
|
||||
try {
|
||||
const remaining = observation.deadlineMs - now();
|
||||
assert.ok(remaining > 0, 'H5 streaming observation ceiling exceeded');
|
||||
await poll(read, remaining, target);
|
||||
} catch (error) { observation.censored = true; onOutcome(result); throw readError ?? error; }
|
||||
}
|
||||
if (readError) throw readError;
|
||||
observation.complete = true; observation.completedAtMs = now(); onOutcome(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function originalTerminal(streaming, terminal) {
|
||||
return streaming.original.status === 'passed'
|
||||
? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null : 'original_terminal_deadline',
|
||||
startedAtMs: terminal.startedAtMs, deadlineMs: terminal.deadlineMs, assessedAtMs: terminal.assessedAtMs }
|
||||
: { status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null };
|
||||
}
|
||||
|
||||
export function assessH5Arm(report, owner) {
|
||||
assert.equal(report.protocolVersion, H5_PROTOCOL);
|
||||
assert.equal(report.observationMode, H5_MODE);
|
||||
assert.equal(report.observationComplete, true, 'H5 requires complete finite observation');
|
||||
assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false);
|
||||
assert.equal(report.diagnosticOnly, false); assert.equal(report.performanceAcceptanceEligible, true);
|
||||
assert.equal(report.cleanup?.complete, true); assert.deepEqual(report.cleanup.errors, []);
|
||||
assert.equal(report.workloadSha256, H5_WORKLOAD_SHA256, 'H5 registration changed');
|
||||
assert.deepEqual(report.observationLimits, H5_LIMITS);
|
||||
assert.deepEqual(report.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(report.actualPlaywrightTimeoutMs, H5_LIMITS.fullInvocationMs);
|
||||
assert.deepEqual(report.cases.map(item => item.name), H5_WORKLOADS.map(item => item.name), 'Complete fixed six-case observation required');
|
||||
assert.equal(owner.status, 'passed', 'Owning Playwright attempt must pass observation/correctness');
|
||||
assert.equal(owner.retry, 0, 'Retries cannot replace an H5 arm');
|
||||
assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, H5_LIMITS.fullInvocationMs);
|
||||
assert.ok(Number.isFinite(owner.durationMs) && owner.durationMs > 0 && owner.durationMs < H5_LIMITS.fullInvocationMs);
|
||||
assert.ok(typeof owner.startedAt === 'string' && Number.isFinite(Date.parse(owner.startedAt)));
|
||||
const misses = [];
|
||||
for (const item of report.cases) {
|
||||
const value = item.h5;
|
||||
assert.ok(value, 'Missing H5 case outcomes');
|
||||
assert.equal(value.streaming.observation.complete, true);
|
||||
assert.equal(value.streaming.observation.censored, false);
|
||||
const streaming = value.streaming.original;
|
||||
for (const field of ['startedAtMs', 'deadlineMs', 'assessedAtMs', 'lastObservedAtMs'])
|
||||
assert.ok(Number.isSafeInteger(streaming[field]) && streaming[field] > 0, 'Missing raw original-deadline timing');
|
||||
assert.ok(['passed', 'failed'].includes(streaming.status));
|
||||
assert.equal(streaming.deadlineMs, streaming.startedAtMs + ORIGINAL_BUDGETS.streamingMs);
|
||||
assert.equal(streaming.target, item.stop ? 3 : item.events);
|
||||
assert.ok(Number.isSafeInteger(streaming.lastObservedCount) && streaming.lastObservedCount >= 0);
|
||||
assert.ok(streaming.lastObservedAtMs >= streaming.startedAtMs && streaming.lastObservedAtMs <= streaming.assessedAtMs);
|
||||
if (streaming.status === 'failed') {
|
||||
assert.equal(streaming.reason, 'original_streaming_deadline');
|
||||
assert.ok(streaming.lastObservedCount < streaming.target);
|
||||
// The original matcher outcome is authoritative; assessedAtMs retains
|
||||
// the actual return time even when it precedes the nominal deadlineMs.
|
||||
assert.equal(item.stop, false, 'Control readiness cannot inherit a streaming SLA miss');
|
||||
assert.ok(streaming.error.includes('Timeout 120000ms exceeded while waiting on the predicate'));
|
||||
assert.equal(value.terminalOriginal.status, 'not-reached');
|
||||
assert.equal(value.terminalOriginal.reason, 'original_streaming_deadline');
|
||||
misses.push(`${item.name}:streaming120`);
|
||||
} else {
|
||||
assert.ok(streaming.lastObservedCount >= streaming.target);
|
||||
assert.equal(streaming.reason, null);
|
||||
assert.equal(streaming.error, null);
|
||||
assert.equal(value.terminalOriginal.status, 'passed');
|
||||
}
|
||||
assert.equal(value.terminalObservation.complete, true, 'H5 terminal observation must drain');
|
||||
assert.equal(value.terminalObservation.censored, false);
|
||||
assert.equal(value.terminalObservation.deadlineMs, value.terminalObservation.startedAtMs + H5_LIMITS.terminalDrainageMs);
|
||||
assert.equal(value.terminalObservation.assessedAtMs, item.settledAtMs);
|
||||
assert.equal(value.terminalObservation.observedRuns, item.runs); assert.equal(value.terminalObservation.terminalRuns, item.runs);
|
||||
assert.ok(value.terminalObservation.lastObservedAtMs >= value.terminalObservation.startedAtMs
|
||||
&& value.terminalObservation.lastObservedAtMs <= value.terminalObservation.assessedAtMs);
|
||||
assert.deepEqual(value.terminalObservation.lastStatuses.map(row => row.runId).sort(), [...item.runIds].sort());
|
||||
assert.ok(value.terminalObservation.lastStatuses.every(row => row.status === (item.stop ? 'cancelled' : 'succeeded')));
|
||||
assert.equal(value.streaming.observation.ceilingMs, H5_LIMITS.streamingMs);
|
||||
assert.equal(value.streaming.observation.deadlineMs, streaming.startedAtMs + H5_LIMITS.streamingMs);
|
||||
assert.ok(value.streaming.observation.completedAtMs >= streaming.assessedAtMs
|
||||
&& value.streaming.observation.completedAtMs <= value.streaming.observation.deadlineMs);
|
||||
assert.ok(value.terminalObservation.startedAtMs >= value.streaming.observation.completedAtMs);
|
||||
assert.ok(item.settledAtMs <= value.terminalObservation.deadlineMs);
|
||||
assert.deepEqual(value.terminalOriginal, originalTerminal(value.streaming, value.terminalObservation));
|
||||
assert.equal(value.visibleCountByOriginalPaintDeadline, item.visible.filter(marker => marker.visibleMs <= streaming.deadlineMs).length);
|
||||
assert.deepEqual(item.terminalStatuses.map(row => row.runId).sort(), [...item.runIds].sort());
|
||||
assert.ok(item.terminalStatuses.every(row => row.runtimeMode === 'native' && row.status === (item.stop ? 'cancelled' : 'succeeded')),
|
||||
'Failed terminal runs are not successful drainage');
|
||||
assert.ok(item.durable.every(row => row.resultAcceptedEvents === (item.stop ? 0 : 1)), 'Every completed run needs its independently persisted result');
|
||||
assert.ok(value.independentDrain.verifiedAtMs >= item.settledAtMs);
|
||||
assert.equal(value.independentDrain.elapsedFromEpochMs, value.independentDrain.verifiedAtMs - item.epochMs);
|
||||
assert.equal(value.independentDrain.deltaCount, item.durable.reduce((sum, row) => sum + row.individualDeltas, 0));
|
||||
assert.equal(value.independentDrain.resultAcceptedCount, item.stop ? 0 : item.runs);
|
||||
assert.equal(value.independentDrain.clockStreamsClosed, true);
|
||||
}
|
||||
if (owner.durationMs >= ORIGINAL_BUDGETS.fullInvocationMs) misses.push('invocation:observed-total-duration900');
|
||||
return { originalSlaPassed: misses.length === 0, misses,
|
||||
original900budget: { status: owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs ? 'passed' : 'failed',
|
||||
assessedAgainst: 'observed owning Playwright total duration', durationMs: owner.durationMs,
|
||||
limitMs: ORIGINAL_BUDGETS.fullInvocationMs,
|
||||
scope: 'Independent conservative gate; total duration can include hooks/teardown that the original timeout treats differently; not an exact original timer-expiration fact' } };
|
||||
}
|
||||
|
||||
export function requireH5Candidate(policy, baseline, candidate) {
|
||||
if (policy === 'final-b2') assert.deepEqual(candidate.misses, [], 'Final B2 must pass every original SLA in every arm');
|
||||
else if (policy === 'cursor-nonregression') {
|
||||
const allowed = new Set(H5_REGISTRATION.knownBaselineMisses);
|
||||
assert.ok(candidate.misses.every(miss => allowed.has(miss) && baseline.misses.includes(miss)),
|
||||
'Cursor-only candidate may inherit only the paired baseline exact registered known misses');
|
||||
} else throw new Error('Explicit final-b2 or cursor-nonregression candidate policy required');
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import { hostname } from "node:os";
|
||||
import path from "node:path";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { json } from "./agent-chat.shared";
|
||||
import { h5Enabled, H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS,
|
||||
observeStreaming, originalTerminal, prepareParticipantAgents, assertParticipantTopology } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
type SeenMarker = { markerCode: string; index: number; visibleMs: number };
|
||||
type Emission = { nonce: string; markerCode: string; index: number; intendedMs: number; emittedMs: number; receivedAtMs: number };
|
||||
@@ -23,6 +25,7 @@ type SetupActivation = { index: number; issueId: string; token: string; waveInde
|
||||
type SetupWave = { index: number; indices: number[]; startedAtMs: number; settledAtMs?: number;
|
||||
statuses?: Array<"fulfilled" | "rejected"> };
|
||||
const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1";
|
||||
const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE);
|
||||
const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "sandbox-remote-with-local-stop";
|
||||
const providerReadinessMs = 300_000;
|
||||
const setupActivationConcurrency = 2;
|
||||
@@ -150,6 +153,7 @@ const processAbsent = (pid: number) => {
|
||||
};
|
||||
|
||||
test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => {
|
||||
const bodyStartedAtMs = Date.now();
|
||||
const pending = new Map<string, Map<string, ServerResponse>>();
|
||||
const clocks = new Map<string, ClockCase>();
|
||||
let markerCounter = 0;
|
||||
@@ -271,10 +275,14 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video),
|
||||
screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" };
|
||||
const report: Record<string, unknown> = {
|
||||
protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport,
|
||||
protocolVersion: h5 ? H5_PROTOCOL : H4D_PROTOCOL, transport, bodyStartedAtMs,
|
||||
agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: Math.max(...cases.map(item => item.runs)) },
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS,
|
||||
observationComplete: false, actualPlaywrightTimeoutMs: info.timeout,
|
||||
originalInvocationBudgetAssessment: "Requires independent owning Playwright observed total duration; conservative original900 assessment, not exact timeout expiration" } : {}),
|
||||
transportPolicy: { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" },
|
||||
resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000,
|
||||
resourceBudgets: h5 ? ORIGINAL_BUDGETS : { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000,
|
||||
terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 },
|
||||
admissionProtocol: "two-at-a-time-provider-and-clock-readiness-before-common-epoch",
|
||||
setupActivationConcurrency,
|
||||
@@ -307,7 +315,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex");
|
||||
const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs",
|
||||
"fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs",
|
||||
"fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs"];
|
||||
"fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs",
|
||||
"fixtures/native-ingestion-h5.mjs"];
|
||||
const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) })));
|
||||
report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex");
|
||||
const harnessRoot = path.resolve(import.meta.dirname, "../..");
|
||||
@@ -349,9 +358,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
expect(localEnvironments).toHaveLength(1);
|
||||
const localEnvironment = localEnvironments[0];
|
||||
expect(localEnvironment.driver).toBe("local");
|
||||
const agents: Array<{ id: string; name: string }> = [];
|
||||
const agentCount = Math.ceil(Math.max(...cases.map(item => item.runs)) / 50);
|
||||
for (let index = 0; index < agentCount; index++) {
|
||||
const agentCount = Math.max(...cases.map(item => item.runs));
|
||||
const agents = await prepareParticipantAgents({ count: agentCount,
|
||||
onOutcome: (value: unknown) => { (report.remote as any).agentSetup = value; },
|
||||
createAndReady: async (index: number) => {
|
||||
const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: {
|
||||
name: `Ingestion ${index + 1}`, role: "engineer", adapterType: "paperclip_runner",
|
||||
defaultEnvironmentId: remoteEnvironment.id,
|
||||
@@ -361,8 +371,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
} }));
|
||||
await expect.poll(async () => (await json(await request.get(`/api/agents/${agent.id}`))).lifecycleState,
|
||||
{ timeout: 60_000 }).toBe("ready");
|
||||
agents.push(agent);
|
||||
}
|
||||
return agent;
|
||||
} }) as Array<{ id: string; name: string }>;
|
||||
(report.remote as any).agentIds = agents.map(agent => agent.id);
|
||||
const controlAgent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: {
|
||||
name: "Ingestion local Stop control", role: "engineer", adapterType: "paperclip_runner",
|
||||
@@ -429,7 +439,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
register(backgroundToken, backgroundCode);
|
||||
const background = await json(await request.post(`/api/companies/${company.id}/issues`, { data: {
|
||||
title: `Ingestion background ${key} ${index}`, description: prompt(backgroundToken, backgroundCode, 0),
|
||||
status: "backlog", assigneeAgentId: caseAgents[Math.floor(index / 50)].id,
|
||||
status: "backlog", assigneeAgentId: caseAgents[index].id,
|
||||
} }));
|
||||
issueIds.push(background.id);
|
||||
issuedTokens.set(background.id, backgroundToken);
|
||||
@@ -508,6 +518,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
appRoot, companyId: company.id, environmentId: caseEnvironment.id, issueIds, transport: caseTransport,
|
||||
});
|
||||
active.controllerAuthority = controllerAuthority;
|
||||
assertParticipantTopology(report, { ...workload, stop, controllerAuthority });
|
||||
const providerClocks = controllerAuthority.rows.map((row: any) => {
|
||||
const token = issuedTokens.get(row.issue_id)!, stream = state.streams.get(token)!;
|
||||
return { runId: row.run_id, nonce: token, launcher: stream.launcher, providerPid: stream.providerPid };
|
||||
@@ -529,7 +540,12 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
active.participants = participants;
|
||||
for (const response of pending.get(key)!.values()) response.writeHead(200, { "Content-Type": "application/json" }).end(JSON.stringify({ epochMs }));
|
||||
const observed = () => page.evaluate(code => (window as unknown as { nativeIngestionVisible: SeenMarker[] }).nativeIngestionVisible.filter(item => item.markerCode === code), markerCode);
|
||||
await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events);
|
||||
let streamingObservation: Awaited<ReturnType<typeof observeStreaming>> | undefined;
|
||||
if (h5) streamingObservation = await observeStreaming({ readCount: async () => (await observed()).length,
|
||||
target: stop ? 3 : workload.events,
|
||||
poll: (read: () => Promise<number>, timeout: number, target: number) => expect.poll(read, { timeout }).toBeGreaterThanOrEqual(target),
|
||||
onOutcome: (value: unknown) => { active.streamingObservation = value; } });
|
||||
else await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events);
|
||||
let stopRequestedAt: number | undefined;
|
||||
if (stop) {
|
||||
const button = page.getByRole("button", { name: "Stop", exact: true });
|
||||
@@ -539,11 +555,25 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
}
|
||||
active.phase = "terminal-drainage";
|
||||
let runs: any[] = [];
|
||||
await expect.poll(async () => {
|
||||
const terminalStartedAtMs = h5 ? Date.now() : undefined;
|
||||
const terminalObservation = h5 ? { startedAtMs: terminalStartedAtMs!, deadlineMs: terminalStartedAtMs! + H5_LIMITS.terminalDrainageMs,
|
||||
assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0,
|
||||
lastStatuses: [] as Array<{ runId: string; status: string }>, complete: false, censored: false } : undefined;
|
||||
if (h5) active.terminalObservation = terminalObservation;
|
||||
try { await expect.poll(async () => {
|
||||
runs = (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId));
|
||||
if (terminalObservation) Object.assign(terminalObservation, { lastObservedAtMs: Date.now(), observedRuns: runs.length,
|
||||
terminalRuns: runs.filter(run => ["succeeded", "cancelled", "failed"].includes(run.status)).length,
|
||||
lastStatuses: runs.map(run => ({ runId: run.id, status: run.status })) });
|
||||
return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status));
|
||||
}, { timeout: 180_000 }).toBe(true);
|
||||
}, { timeout: 180_000 }).toBe(true); }
|
||||
catch (error) {
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = Date.now(); terminalObservation.censored = true;
|
||||
active.terminalOriginal = originalTerminal(streamingObservation!, terminalObservation); }
|
||||
throw error;
|
||||
}
|
||||
const settledAt = Date.now(), seenMarkers = await observed();
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = settledAt; terminalObservation.complete = true; }
|
||||
active.settledAt = settledAt; active.seenMarkers = seenMarkers;
|
||||
const details = [];
|
||||
for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`)));
|
||||
@@ -618,6 +648,11 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs };
|
||||
const clockEvidence: Array<Record<string, unknown>> = [];
|
||||
const result = { ...workload, stop, transport: caseTransport, environmentId: caseEnvironment.id,
|
||||
...(h5 ? { h5: { streaming: streamingObservation, terminalObservation,
|
||||
terminalOriginal: originalTerminal(streamingObservation!, terminalObservation!),
|
||||
visibleCountByOriginalPaintDeadline: seenMarkers.filter(marker => marker.visibleMs <= streamingObservation!.original.deadlineMs).length,
|
||||
deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : {}),
|
||||
...(h5 ? { terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status })) } : {}),
|
||||
setupActivationConcurrency, setupReadinessStartedAtMs, readinessDeadlineMs, allReadyObservedAtMs, activations, activationWaves,
|
||||
nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt,
|
||||
stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps,
|
||||
@@ -648,16 +683,19 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const code = state.streams.get(token)!.markerCode;
|
||||
expect(token).toBeTruthy();
|
||||
const deltas: Array<{ index: number; identity: string }> = [];
|
||||
let afterSeq = 0;
|
||||
let afterSeq = 0, resultAcceptedEvents = 0;
|
||||
while (true) {
|
||||
const rows = await json(await request.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
|
||||
if (!rows.length) break;
|
||||
for (const row of rows) {
|
||||
const event = row.payload?.prpEvent;
|
||||
if (h5 && (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted")) resultAcceptedEvents++;
|
||||
if (event?.eventType !== "item.delta") continue;
|
||||
for (const match of String(event.payload?.text ?? "").matchAll(new RegExp(`\\[${code}([a-z0-9]{1,3})\\]`, "g"))) {
|
||||
const index = Number.parseInt(match[1], 36);
|
||||
expect(event.payload.text, "Every durable normalized delta retains its complete compact payload").toBe(`[${code}${index.toString(36)}] `);
|
||||
if (h5) { expect(event.runId).toBe(run.id); expect(event.sourceInstanceId).toBeTruthy();
|
||||
expect(event.sourceEventId).toBeTruthy(); expect(Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > 0).toBe(true); }
|
||||
deltas.push({ index, identity: `${event.sourceInstanceId}:${event.sourceEventId}:${event.sourceSeq}` });
|
||||
}
|
||||
}
|
||||
@@ -674,11 +712,21 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
expect(new Set(deltas.map(item => item.identity)).size).toBe(deltas.length);
|
||||
expect(deltas.map(item => item.index)).toEqual(Array.from({ length: stop ? deltas.length : workload.events }, (_, index) => index));
|
||||
if (stop) for (const item of visible) expect(deltas.some(delta => delta.index === item.index)).toBe(true);
|
||||
durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, matchedSourceClocks: deltas.length, ordered: true });
|
||||
if (h5) expect(resultAcceptedEvents).toBe(stop ? 0 : 1);
|
||||
durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length,
|
||||
matchedSourceClocks: deltas.length, ordered: true, ...(h5 ? { resultAcceptedEvents } : {}) });
|
||||
}
|
||||
await expect.poll(() => [...state.streams.values()].every(stream => stream.ended || stream.aborted), { timeout: 30_000 }).toBe(true);
|
||||
expect(state.errors).toEqual([]);
|
||||
expect([...state.streams.values()].every(stream => !stream.error)).toBe(true);
|
||||
if (h5) {
|
||||
const verifiedAtMs = Date.now();
|
||||
Object.assign(result.h5!, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs,
|
||||
deltaCount: durable.reduce((count, row) => count + row.individualDeltas, 0),
|
||||
resultAcceptedCount: durable.reduce((count, row) => count + (row.resultAcceptedEvents ?? 0), 0),
|
||||
clockStreamsClosed: true,
|
||||
scope: "Independent completion upper bound after existing reload/API/clock/lease checks; includes verification and screenshot overhead, not exact last-commit time" } });
|
||||
}
|
||||
clockEvidence.push(...[...state.streams.values()].map(stream => ({ nonce: stream.nonce, markerCode: stream.markerCode,
|
||||
launcher: stream.launcher, providerPid: stream.providerPid,
|
||||
hello: stream.hello, ended: stream.ended, aborted: stream.aborted, error: stream.error, partialBytes: stream.partialBytes,
|
||||
@@ -691,6 +739,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
for (const workload of cases) await perform(workload);
|
||||
await perform({ name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0 }, true);
|
||||
report.complete = true;
|
||||
if (h5) report.observationComplete = true;
|
||||
} catch (error) {
|
||||
report.primaryFailure = { name: error instanceof Error ? error.name : "UnknownError",
|
||||
message: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined };
|
||||
@@ -728,7 +777,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
await attemptCleanup("restore-experimental-settings", async () => json(await request.patch("/api/instance/settings/experimental", {
|
||||
data: { enableNativeRunner: original.enableNativeRunner, enableWorktreeRunExecution: original.enableWorktreeRunExecution } })));
|
||||
report.cleanup = { errors: cleanupErrors, complete: cleanupErrors.length === 0 };
|
||||
if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; }
|
||||
if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false;
|
||||
if (h5) report.observationComplete = false; }
|
||||
if (h5) report.bodyFinishedAtMs = Date.now();
|
||||
await info.attach("native-ingestion-evidence", { body: JSON.stringify(report, null, 2), contentType: "application/json" });
|
||||
if (!report.primaryFailure) expect(cleanupErrors, "API cleanup must succeed before accepting an invocation").toEqual([]);
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { defineConfig, type ReporterDescription } from "@playwright/test";
|
||||
import base from "./playwright.config";
|
||||
import { h5Enabled, H5_LIMITS } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
const server = base.webServer as Exclude<typeof base.webServer, unknown[]>;
|
||||
const evidence = process.env.PAPERCLIP_INGESTION_EVIDENCE_DIR;
|
||||
const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE);
|
||||
const browserRecording = Boolean(process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR) || process.env.PAPERCLIP_INGESTION_BROWSER_CAPTURE === "1";
|
||||
process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "off";
|
||||
const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../.."));
|
||||
@@ -33,7 +35,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-"
|
||||
export default defineConfig({
|
||||
...base,
|
||||
testMatch: "native-event-ingestion.journey.ts",
|
||||
timeout: 900_000,
|
||||
timeout: h5 ? H5_LIMITS.fullInvocationMs : 900_000,
|
||||
...(evidence ? { outputDir: path.join(evidence, "test-results"),
|
||||
reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}),
|
||||
use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 },
|
||||
|
||||
Reference in new issue
Block a user