diff --git a/scripts/compare-native-ingestion-browser.ts b/scripts/compare-native-ingestion-browser.ts index 354e7b0ef37..d593deb41db 100644 --- a/scripts/compare-native-ingestion-browser.ts +++ b/scripts/compare-native-ingestion-browser.ts @@ -3,6 +3,9 @@ import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; +import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS, + assessH5Arm, requireH5Candidate, assertParticipantSetup, assertParticipantTopology } from "../tests/e2e/fixtures/native-ingestion-h5.mjs"; +import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs"; // Each manifest entry points at native-ingestion-evidence JSON attachments // from two isolated compiled-app runs. Run the variants in alternating order; @@ -11,12 +14,88 @@ const arg = (name: string) => process.argv.find(value => value.startsWith(`--${n const manifestPath = arg("pairs"); assert.ok(manifestPath, "Pass --pairs="); const phase = arg("phase") ?? "compare", expectation = arg("expect") ?? "improvement"; +const h5 = arg("observation-mode") === H5_MODE; +assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode"); +const candidatePolicy = arg("candidate-policy"); +assert.ok(!arg("original-h4c-proof") && !arg("original-h4c-inventory"), "Old H4c proof arguments cannot confirm the new independent-agent geometry"); assert.ok(["aa", "compare"].includes(phase)); assert.ok(["improvement", "nonregression"].includes(expectation)); -const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string }>; +const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string; + baselinePlaywrightFile?: string; candidatePlaywrightFile?: string; baselineInventoryFile?: string; candidateInventoryFile?: string }>; assert.ok(manifest.length >= 5, "At least five actual paired application runs are required"); +if (h5) assert.equal(manifest.length, 5, "H5 requires exactly five prospectively registered pairs"); const load = async (file: string) => JSON.parse(await readFile(path.resolve(path.dirname(manifestPath), file), "utf8")); -const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await load(entry.baselineFile), candidate: await load(entry.candidateFile) }))); +async function owningResult(reportPath: string, evidence?: any) { + const bytes = await readFile(reportPath); + const report = JSON.parse(bytes.toString("utf8")), tests: any[] = []; + assert.deepEqual(report.errors ?? [], [], "Owning Playwright report has a global error"); + const visit = (value: any) => { + if (Array.isArray(value)) for (const child of value) visit(child); + else if (value && typeof value === "object") { + if (Array.isArray(value.results) && Object.hasOwn(value, "expectedStatus")) tests.push(value); + for (const [key, child] of Object.entries(value)) if (key !== "results") visit(child); + } + }; + visit(report); assert.equal(tests.length, 1, "One complete owning test is required; no partial selection"); + const test = tests[0]; assert.equal(test.results.length, 1, "No retry or omitted owning attempt"); + assert.equal(test.expectedStatus, "passed"); + const result = test.results[0], attachments = result.attachments.filter((item: any) => item.name === "native-ingestion-evidence"); + assert.ok(result.error === undefined || result.error === null, "Owning Playwright attempt has an error"); + assert.equal(attachments.length, 1, "Owning result must carry this exact evidence attachment"); + const attachment = attachments[0]; + const attached = attachment.body ? JSON.parse(Buffer.from(attachment.body, "base64").toString("utf8")) + : JSON.parse(await readFile(path.resolve(path.dirname(reportPath), attachment.path), "utf8")); + if (evidence) assert.deepEqual(attached, evidence, "Playwright result belongs to a different evidence arm"); + return { evidence: attached, owner: { reportFile: reportPath, reportSha256: createHash("sha256").update(bytes).digest("hex"), + status: result.status, retry: result.retry, errors: result.errors, startedAt: result.startTime, + durationMs: result.duration, timeoutMs: test.timeout } }; +} +async function loadArm(file: string, playwrightFile?: string, inventoryFile?: string) { + const evidence = await load(file); + if (!h5) return evidence; + assert.ok(playwrightFile, "H5 requires each arm's independent owning Playwright JSON report"); + evidence.owningPlaywright = (await owningResult(path.resolve(path.dirname(manifestPath!), playwrightFile), evidence)).owner; + assert.ok(inventoryFile, "H5 requires each arm's exact full artifact inventory sidecar"); + evidence.armInventory = await validateArmInventory(path.resolve(path.dirname(manifestPath!), inventoryFile), + { evidence, owningPlaywright: evidence.owningPlaywright }); + return evidence; +} +const pairs = await Promise.all(manifest.map(async entry => ({ baseline: await loadArm(entry.baselineFile, entry.baselinePlaywrightFile, entry.baselineInventoryFile), + candidate: await loadArm(entry.candidateFile, entry.candidatePlaywrightFile, entry.candidateInventoryFile) }))); +const h5Assessments: any[] = []; +if (h5) { + if (phase === "compare") { + assert.ok(["final-b2", "cursor-nonregression"].includes(candidatePolicy ?? ""), "H5 requires an explicit candidate SLA policy"); + assert.equal(expectation, candidatePolicy === "cursor-nonregression" ? "nonregression" : "improvement"); + } + for (const pair of pairs) { + assert.ok(Object.values(H5_BASELINE_REVISIONS).includes(pair.baseline.sourceRevision), "H5 baseline must use the prospectively frozen B0 or B1 revision"); + const baselineVariant = pair.baseline.sourceRevision === H5_BASELINE_REVISIONS.B0 ? "B0" : "B1"; + assert.equal(pair.baseline.armInventory.variant, baselineVariant, "Baseline inventory role differs from its frozen revision"); + if (phase === "aa") assert.equal(pair.candidate.armInventory.variant, baselineVariant, "A/A must use the same baseline inventory role"); + if (phase === "compare" && candidatePolicy === "cursor-nonregression") { + assert.equal(pair.baseline.sourceRevision, H5_BASELINE_REVISIONS.B0, "Cursor policy requires B0→B1"); + assert.equal(pair.candidate.sourceRevision, H5_BASELINE_REVISIONS.B1, "Cursor policy requires B0→B1"); + assert.equal(pair.candidate.armInventory.variant, "B1", "Cursor policy cannot stand in for final B2 acceptance"); + } else if (phase === "compare") { + assert.equal(pair.candidate.armInventory.variant, "B2", "Final policy requires the qualified B2 inventory role"); + assert.ok(!Object.values(H5_BASELINE_REVISIONS).includes(pair.candidate.sourceRevision), "Final B2 must differ from both frozen baselines"); + } + const baseline = assessH5Arm(pair.baseline, pair.baseline.owningPlaywright); + const candidate = assessH5Arm(pair.candidate, pair.candidate.owningPlaywright); + // The only prospectively known baseline workload SLA miss is loaded streaming. + const allowed = new Set(["burst512:streaming120", "invocation:observed-total-duration900"]); + for (const miss of baseline.misses) assert.ok(allowed.has(miss), "Unknown baseline deadline failure needs diagnosis"); + if (phase === "aa") for (const miss of candidate.misses) assert.ok(allowed.has(miss), "Unknown A/A deadline failure needs diagnosis"); + let candidateSlaPolicyError; + if (phase === "compare") { + try { requireH5Candidate(candidatePolicy, baseline, candidate); } + catch (error) { candidateSlaPolicyError = error instanceof Error ? error.message : String(error); } + } + h5Assessments.push({ baseline, candidate, candidateSlaPolicyError, candidateSlaPolicyPassed: !candidateSlaPolicyError, + owningPlaywright: { baseline: pair.baseline.owningPlaywright, candidate: pair.candidate.owningPlaywright } }); + } +} const workloads = [ { name: "paced", runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false }, { name: "burst16", runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false }, @@ -30,9 +109,128 @@ const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floo const first = pairs[0]; const sha256 = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); const uuid = (value: unknown) => typeof value === "string" && /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value); +function assertCaseAuthority(report: any, result: any) { + const remote = report.remote, local = report.local; + const authority = result.controllerAuthority; + assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing controller authority proof before the measured epoch"); + assert.equal(authority?.kind, result.stop ? "host-file-journal" : "database-controller-checkpoint", "Wrong checkpoint authority for actual transport"); + assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction"); + assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation"); + assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0 + && Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0 + && authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs, + "Close the actual remote authority snapshot before the measured epoch"); + assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof"); + assertParticipantTopology(report, result); + assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch"); + assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority"); + const authorityByRun = new Map(); + for (const row of authority.rows) { + assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual task/lease UUID"); + assert.equal(row.company_id, report.companyId, "Remote authority company mismatch"); + assert.equal(row.environment_id, result.environmentId, "Remote authority environment mismatch"); + assert.equal(row.runtime_mode, "native"); + if (!result.stop) { + assert.ok(remote.agentIds.includes(row.agent_id), "Remote run did not use its normal remote agent"); + assert.equal(row.provider, "fake-plugin"); + assert.match(row.provider_lease_id, /^fake-plugin:\/\//); + assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header"); + assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0); + } else { + assert.equal(row.agent_id, local.controlAgentId, "Stop did not use its separate local control agent"); + assert.equal(row.provider, "local", "Stop requires its normal local lease"); + assert.equal(row.provider_lease_id, null); assert.equal(row.checkpoint_part, null, "Local Stop must retain host-file authority without a database header"); + assert.ok(BigInt(String(row.process_identity_seq)) > 0n); + assert.equal(row.process_identity_source_event_id, null, "Local identity must be server-authored"); + assert.equal(row.process_identity?.targetKind, "local"); + assert.equal(row.process_identity?.processPid, row.process_pid); + assert.equal(row.process_identity?.processGroupId, row.process_group_id); + assert.equal(row.process_identity?.processStartedAt, row.process_started_at); + assert.ok(uuid(row.admitted_execution_workspace_id), "Missing actual admitted local workspace binding"); + } + authorityByRun.set(row.run_id, row); + } + return { authority, authorityByRun }; +} + +function assertCaseSettlement(report: any, result: any, validated = assertCaseAuthority(report, result)) { + const remote = report.remote, local = report.local; + const { authority, authorityByRun } = validated; + if (result.stop) { + assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed"); + const row = authority.rows[0], journal = authority.localJournal, before = result.localProcesses; + assert.equal(journal?.schema, "paperclip.runner.durable.control-plane-state.v1", "Missing owned local journal authority"); + assert.ok(sha256(journal.sha256) && journal.bytes > 0 && Number.isSafeInteger(journal.checkedAtMs) + && journal.checkedAtMs < result.epochMs, "Verify actual local file journal before the measured epoch"); + assert.notEqual(journal.checkpointEventPolicy, "control-v1", "Preserve the original local raw transport recovery policy"); + for (const [field, expected] of [["runId", row.run_id], ["runnerInstanceId", row.runner_instance_id], + ["normalizedSessionId", row.native_session_id], ["environmentLeaseId", row.admitted_execution_workspace_id]]) assert.equal(journal.identity?.[field], expected, "Local journal identity mismatch"); + assert.ok(before && before.hostname === row.process_identity.hostname && before.runnerPid === row.process_pid + && before.processGroupId === row.process_group_id && before.providerPid === result.providerClocks[0].providerPid, + "Local observed process ownership mismatch"); + assert.ok(before.runnerAlive && before.groupAlive && before.providerAlive && Number.isSafeInteger(before.checkedAtMs) + && before.checkedAtMs < result.epochMs, "Independently observe all owned local processes before emission"); + const stopped = result.localStopProof, database = stopped?.database, after = database?.rows?.[0]; + assert.ok(stopped && Number.isSafeInteger(stopped.checkedAtMs) && stopped.checkedAtMs >= result.stopAcknowledgedAtMs + && stopped.checkedAtMs >= result.settledAtMs, "Verify local physical Stop after timing boundaries"); + assert.equal(database?.kind, "host-file-journal"); assert.equal(database?.observedReadOnly, "on"); + assert.equal(database?.observedIsolation, "repeatable read"); assert.equal(database?.rows?.length, 1); + assert.equal(database?.checkedBeforeMeasuredEpoch, false); + assert.ok(Number.isSafeInteger(database.checkedAtMs) && database.checkedAtMs >= result.stopAcknowledgedAtMs + && Number.isSafeInteger(database.connectionClosedAtMs) && database.connectionClosedAtMs >= database.checkedAtMs + && database.connectionClosedAtMs <= stopped.checkedAtMs, "Local stopped database proof predates acknowledgement"); + for (const field of ["run_id", "company_id", "environment_id", "lease_id", "agent_id", "provider", "provider_lease_id", + "admitted_execution_workspace_id", "process_identity_seq", "process_identity_source_event_id"]) assert.equal(after?.[field], row[field], "Local stopped database identity mismatch"); + assert.equal(after.status, "cancelled"); assert.equal(after.cancellation_dispatch_state, "acknowledged"); + if (!after.process_pid && !after.process_group_id) { + assert.ok(BigInt(String(after.process_stopped_seq)) > BigInt(String(row.process_identity_seq)), "Cleared local process identity needs its server stop receipt"); + assert.equal(after.process_stopped_source_event_id, null); + assert.equal(after.process_stopped?.processPid, before.runnerPid); + assert.equal(after.process_stopped?.processGroupId, before.processGroupId); + assert.equal(after.process_stopped?.processStartedAt, row.process_identity.processStartedAt); + } else { + assert.equal(after.process_pid, before.runnerPid); assert.equal(after.process_group_id, before.processGroupId); + } + for (const field of ["runnerPid", "processGroupId", "providerPid", "hostname"]) assert.equal(stopped.processes?.[field], before[field]); + assert.equal(stopped.processes.runnerAbsent, true); assert.equal(stopped.processes.groupAbsent, true); + assert.equal(stopped.processes.providerAbsent, true, "Local provider remains alive after Stop"); + const lease = stopped.lease; + assert.equal(lease?.id, row.lease_id); assert.equal(lease?.companyId, report.companyId); + assert.equal(lease?.environmentId, local.environmentId); assert.equal(lease?.heartbeatRunId, row.run_id); + assert.equal(lease?.provider, "local"); assert.equal(lease?.providerLeaseId, null); + assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt + && lease.cleanupStatus !== "failed", "Local Stop lease cleanup remains unsettled"); + assert.equal(result.remoteLeases, undefined, "Local Stop cannot claim fake remote cleanup"); + } else { + assert.equal(result.localStopProof, undefined, "Streaming cannot claim a local Stop proof"); + assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof"); + assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch"); + assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs, + "Verify remote leases after recorded timing boundaries"); + for (const lease of result.remoteLeases) { + const row = authorityByRun.get(lease.runId); + assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId); + assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider); + assert.equal(lease.providerLeaseId, row.provider_lease_id); + assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt + && lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled"); + if (lease.terminationReceipt) { + const receipt = lease.terminationReceipt; + assert.equal(receipt.schema, "paperclip.remote-termination.v1"); + assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId); + assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider); + assert.equal(receipt.providerLeaseId, lease.providerLeaseId); + assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt"); + } + } + } +} + + for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) { const report = pair[arm]; - assert.equal(report.protocolVersion, "v4-remote-streaming-original-local-stop-clock-stream", "Fresh browser acceptance requires the qualified v4 mixed clock protocol"); + assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : H4D_PROTOCOL, + "Fresh browser acceptance requires the independent-agent observation or default-deadline protocol"); assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport"); assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local", realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, "Missing explicit transport and real cloud Stop qualification policy"); @@ -83,6 +281,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.equal(report.setupActivationConcurrency, 2, "Keep the qualified setup activation concurrency of two"); assert.equal(report.setupReadinessPollingMs, 50, "Keep the qualified setup readiness observation interval"); assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false, "Smoke cannot qualify performance"); + assertParticipantSetup(report); assert.equal(report.cleanup?.complete, true, "Acceptance requires completed fixture cleanup"); assert.deepEqual(report.cleanup?.errors, [], "Acceptance rejects fixture cleanup errors"); assert.notEqual(report.diagnosticOnly, true, "Diagnostic profiles cannot qualify unprofiled performance"); @@ -97,7 +296,13 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.equal(report.sourceRevision, first[arm].sourceRevision, "Every sample must cover the same revision per arm"); assert.equal(report.serverEntrySha256, first[arm].serverEntrySha256, "Compiled artifact changed between samples"); assert.deepEqual(report.compiledHotPathHashes, first[arm].compiledHotPathHashes, "Compiled ingestion/runtime changed between samples"); + if (h5) for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) + assert.equal(report.armInventory[field], first[arm].armInventory[field], "Full qualified artifact inventory changed between samples"); assert.equal(report.measurementSha256, first.baseline.measurementSha256, "Fixture/observer protocol changed between runs"); + assert.match(report.harnessProvenance?.sourceRevision ?? "", /^[a-f0-9]{40}$/, "Missing frozen harness source revision"); + assert.equal(report.harnessProvenance.sourceStatus, ""); + for (const field of ["sourceRevision", "sourceHashes", "driverLockSha256"]) + assert.deepEqual(report.harnessProvenance[field], first.baseline.harnessProvenance[field], "Harness source changed between arms"); if (phase === "aa") assert.equal(report.sourceRevision, first.baseline.sourceRevision, "A/A must use one source revision"); for (const name of names) { const result = report.cases.find((item: any) => item.name === name); @@ -124,44 +329,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) assert.ok(result.runIds.every(uuid), "Missing actual remote run UUID"); assert.equal(new Set(result.runIds).size, result.runs, "Duplicate remote run identity"); assert.deepEqual(result.durable.map((proof: any) => proof.runId).sort(), [...result.runIds].sort(), "Durable proof run set mismatch"); - const authority = result.controllerAuthority; - assert.equal(authority?.checkedBeforeMeasuredEpoch, true, "Missing controller authority proof before the measured epoch"); - assert.equal(authority?.kind, result.stop ? "host-file-journal" : "database-controller-checkpoint", "Wrong checkpoint authority for actual transport"); - assert.equal(authority?.observedReadOnly, "on", "Remote authority proof must observe a read-only transaction"); - assert.equal(authority?.observedIsolation, "repeatable read", "Remote authority proof must observe repeatable-read isolation"); - assert.ok(Number.isSafeInteger(authority?.checkedAtMs) && authority.checkedAtMs > 0 - && Number.isSafeInteger(authority.connectionClosedAtMs) && authority.connectionClosedAtMs > 0 - && authority.checkedAtMs <= authority.connectionClosedAtMs && authority.connectionClosedAtMs < result.epochMs, - "Close the actual remote authority snapshot before the measured epoch"); - assert.equal(authority.rows?.length, result.runs, "Every remote run needs its controller checkpoint and lease proof"); - assert.deepEqual(authority.rows.map((row: any) => row.run_id).sort(), [...result.runIds].sort(), "Remote authority run set mismatch"); - assert.equal(new Set(authority.rows.map((row: any) => row.issue_id)).size, result.runs, "Duplicate remote issue authority"); - const authorityByRun = new Map(); - for (const row of authority.rows) { - assert.ok(uuid(row.issue_id) && uuid(row.lease_id), "Missing actual task/lease UUID"); - assert.equal(row.company_id, report.companyId, "Remote authority company mismatch"); - assert.equal(row.environment_id, result.environmentId, "Remote authority environment mismatch"); - assert.equal(row.runtime_mode, "native"); - if (!result.stop) { - assert.ok(remote.agentIds.includes(row.agent_id), "Remote run did not use its normal remote agent"); - assert.equal(row.provider, "fake-plugin"); - assert.match(row.provider_lease_id, /^fake-plugin:\/\//); - assert.equal(row.checkpoint_part, "header", "Missing actual database controller checkpoint header"); - assert.ok(Number.isSafeInteger(row.checkpoint_header_bytes) && row.checkpoint_header_bytes > 0); - } else { - assert.equal(row.agent_id, local.controlAgentId, "Stop did not use its separate local control agent"); - assert.equal(row.provider, "local", "Stop requires its normal local lease"); - assert.equal(row.provider_lease_id, null); assert.equal(row.checkpoint_part, null, "Local Stop must retain host-file authority without a database header"); - assert.ok(BigInt(String(row.process_identity_seq)) > 0n); - assert.equal(row.process_identity_source_event_id, null, "Local identity must be server-authored"); - assert.equal(row.process_identity?.targetKind, "local"); - assert.equal(row.process_identity?.processPid, row.process_pid); - assert.equal(row.process_identity?.processGroupId, row.process_group_id); - assert.equal(row.process_identity?.processStartedAt, row.process_started_at); - assert.ok(uuid(row.admitted_execution_workspace_id), "Missing actual admitted local workspace binding"); - } - authorityByRun.set(row.run_id, row); - } + const { authority, authorityByRun } = assertCaseAuthority(report, result); assert.equal(result.providerClocks?.length, result.runs, "Every actual provider needs its wrapper launcher and PID"); assert.deepEqual(result.providerClocks.map((clock: any) => clock.runId).sort(), [...result.runIds].sort()); assert.equal(new Set(result.providerClocks.map((clock: any) => clock.providerPid)).size, result.runs); @@ -223,74 +391,7 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) } } assert.equal(result.settledAtMs, result.epochMs + result.drainThroughTerminalMs, "Terminal timing boundary changed"); - if (result.stop) { - assert.equal(result.stopAcknowledgedAtMs, result.stopRequestedAtMs + result.stopThroughAcknowledgementMs, "Stop acknowledgement timing boundary changed"); - const row = authority.rows[0], journal = authority.localJournal, before = result.localProcesses; - assert.equal(journal?.schema, "paperclip.runner.durable.control-plane-state.v1", "Missing owned local journal authority"); - assert.ok(sha256(journal.sha256) && journal.bytes > 0 && Number.isSafeInteger(journal.checkedAtMs) - && journal.checkedAtMs < result.epochMs, "Verify actual local file journal before the measured epoch"); - assert.notEqual(journal.checkpointEventPolicy, "control-v1", "Preserve the original local raw transport recovery policy"); - for (const [field, expected] of [["runId", row.run_id], ["runnerInstanceId", row.runner_instance_id], - ["normalizedSessionId", row.native_session_id], ["environmentLeaseId", row.admitted_execution_workspace_id]]) assert.equal(journal.identity?.[field], expected, "Local journal identity mismatch"); - assert.ok(before && before.hostname === row.process_identity.hostname && before.runnerPid === row.process_pid - && before.processGroupId === row.process_group_id && before.providerPid === result.providerClocks[0].providerPid, - "Local observed process ownership mismatch"); - assert.ok(before.runnerAlive && before.groupAlive && before.providerAlive && Number.isSafeInteger(before.checkedAtMs) - && before.checkedAtMs < result.epochMs, "Independently observe all owned local processes before emission"); - const stopped = result.localStopProof, database = stopped?.database, after = database?.rows?.[0]; - assert.ok(stopped && Number.isSafeInteger(stopped.checkedAtMs) && stopped.checkedAtMs >= result.stopAcknowledgedAtMs - && stopped.checkedAtMs >= result.settledAtMs, "Verify local physical Stop after timing boundaries"); - assert.equal(database?.kind, "host-file-journal"); assert.equal(database?.observedReadOnly, "on"); - assert.equal(database?.observedIsolation, "repeatable read"); assert.equal(database?.rows?.length, 1); - assert.equal(database?.checkedBeforeMeasuredEpoch, false); - assert.ok(Number.isSafeInteger(database.checkedAtMs) && database.checkedAtMs >= result.stopAcknowledgedAtMs - && Number.isSafeInteger(database.connectionClosedAtMs) && database.connectionClosedAtMs >= database.checkedAtMs - && database.connectionClosedAtMs <= stopped.checkedAtMs, "Local stopped database proof predates acknowledgement"); - for (const field of ["run_id", "company_id", "environment_id", "lease_id", "agent_id", "provider", "provider_lease_id", - "admitted_execution_workspace_id", "process_identity_seq", "process_identity_source_event_id"]) assert.equal(after?.[field], row[field], "Local stopped database identity mismatch"); - assert.equal(after.status, "cancelled"); assert.equal(after.cancellation_dispatch_state, "acknowledged"); - if (!after.process_pid && !after.process_group_id) { - assert.ok(BigInt(String(after.process_stopped_seq)) > BigInt(String(row.process_identity_seq)), "Cleared local process identity needs its server stop receipt"); - assert.equal(after.process_stopped_source_event_id, null); - assert.equal(after.process_stopped?.processPid, before.runnerPid); - assert.equal(after.process_stopped?.processGroupId, before.processGroupId); - assert.equal(after.process_stopped?.processStartedAt, row.process_identity.processStartedAt); - } else { - assert.equal(after.process_pid, before.runnerPid); assert.equal(after.process_group_id, before.processGroupId); - } - for (const field of ["runnerPid", "processGroupId", "providerPid", "hostname"]) assert.equal(stopped.processes?.[field], before[field]); - assert.equal(stopped.processes.runnerAbsent, true); assert.equal(stopped.processes.groupAbsent, true); - assert.equal(stopped.processes.providerAbsent, true, "Local provider remains alive after Stop"); - const lease = stopped.lease; - assert.equal(lease?.id, row.lease_id); assert.equal(lease?.companyId, report.companyId); - assert.equal(lease?.environmentId, local.environmentId); assert.equal(lease?.heartbeatRunId, row.run_id); - assert.equal(lease?.provider, "local"); assert.equal(lease?.providerLeaseId, null); - assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt - && lease.cleanupStatus !== "failed", "Local Stop lease cleanup remains unsettled"); - assert.equal(result.remoteLeases, undefined, "Local Stop cannot claim fake remote cleanup"); - } else { - assert.equal(result.localStopProof, undefined, "Streaming cannot claim a local Stop proof"); - assert.equal(result.remoteLeases?.length, result.runs, "Every remote run needs post-timing lease cleanup proof"); - assert.deepEqual(result.remoteLeases.map((lease: any) => lease.runId).sort(), [...result.runIds].sort(), "Remote cleanup run set mismatch"); - assert.ok(Number.isSafeInteger(result.remoteLeasesCheckedAtMs) && result.remoteLeasesCheckedAtMs >= result.settledAtMs, - "Verify remote leases after recorded timing boundaries"); - for (const lease of result.remoteLeases) { - const row = authorityByRun.get(lease.runId); - assert.equal(lease.id, row.lease_id); assert.equal(lease.companyId, report.companyId); - assert.equal(lease.environmentId, remote.environmentId); assert.equal(lease.provider, row.provider); - assert.equal(lease.providerLeaseId, row.provider_lease_id); - assert.ok(["released", "expired", "failed"].includes(lease.status) && lease.releasedAt - && lease.cleanupStatus === "success", "Remote lease cleanup remains unsettled"); - if (lease.terminationReceipt) { - const receipt = lease.terminationReceipt; - assert.equal(receipt.schema, "paperclip.remote-termination.v1"); - assert.equal(receipt.companyId, report.companyId); assert.equal(receipt.runId, lease.runId); - assert.equal(receipt.leaseId, lease.id); assert.equal(receipt.provider, lease.provider); - assert.equal(receipt.providerLeaseId, lease.providerLeaseId); - assert.ok(["stopped", "destroyed"].includes(receipt.state), "Invalid recorded remote termination receipt"); - } - } - } + assertCaseSettlement(report, result, { authority, authorityByRun }); for (const proof of result.durable) { assert.equal(proof.ordered, true); assert.equal(proof.uniqueSourceIdentities, proof.individualDeltas); assert.equal(proof.matchedSourceClocks, proof.individualDeltas, "Every durable measured delta needs an original provider clock"); @@ -300,6 +401,61 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`); } } +let originalH4dConfirmation: Record | undefined; +if (h5 && phase === "compare" && candidatePolicy === "final-b2") { + originalH4dConfirmation = { required: true, passed: false, reason: "Later H4d default-mode full functional proof on the same source and geometry is pending" }; + if (arg("original-h4d-proof")) { + try { + assert.ok(arg("original-h4d-inventory"), "H4d original-deadline proof requires its full artifact inventory sidecar"); + const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4d-proof")!)); + assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []); + assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs); + assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs); + assert.equal(proof.protocolVersion, H4D_PROTOCOL, "Old H4c geometry cannot confirm H5b"); + assertParticipantSetup(proof); + assert.deepEqual(proof.agentTopology, first.candidate.agentTopology); + assert.equal(proof.observationMode, undefined, "Confirmation must use H4d default deadlines"); + assert.equal(proof.harnessProvenance?.sourceRevision, first.candidate.harnessProvenance.sourceRevision, "H4d confirmation must use the same frozen harness source"); + assert.deepEqual(proof.harnessProvenance.sourceHashes, first.candidate.harnessProvenance.sourceHashes); + assert.equal(proof.harnessProvenance.driverLockSha256, first.candidate.harnessProvenance.driverLockSha256); + assert.equal(proof.harnessProvenance.sourceStatus, ""); + assert.equal(proof.measurementSha256, first.candidate.measurementSha256, "H4d confirmation must bind the same measured harness bytes"); + assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS); + assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false); + assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled"); + assert.equal(proof.cleanup?.complete, true); assert.deepEqual(proof.cleanup.errors, []); + assert.deepEqual(proof.browserRecording, first.candidate.browserRecording); + assert.equal(proof.sourceRevision, first.candidate.sourceRevision); assert.equal(proof.compiledBuildRevision, first.candidate.compiledBuildRevision); + assert.equal(proof.serverEntrySha256, first.candidate.serverEntrySha256); + assert.deepEqual(proof.compiledHotPathHashes, first.candidate.compiledHotPathHashes); + assert.equal(proof.transport, first.candidate.transport); assert.deepEqual(proof.transportPolicy, first.candidate.transportPolicy); + assert.deepEqual(proof.cases.map((item: any) => item.name), names, "Original confirmation must include all six workloads"); + for (const fixed of workloads) { + const item = proof.cases.find((item: any) => item.name === fixed.name); + assertCaseSettlement(proof, item); + for (const field of ["runs", "events", "intervalMs", "prelude", "stop"] as const) assert.equal(item[field], fixed[field]); + assert.equal(item.persistedAfterReload, true); assert.equal(item.durable.length, fixed.runs); + assert.equal(item.matchedVisibleClocks, item.visible.length); + assert.ok(item.visible.length >= (fixed.stop ? 3 : fixed.events)); + assert.equal(item.runIds.length, fixed.runs); assert.equal(new Set(item.runIds).size, fixed.runs); + for (const row of item.durable) { + assert.equal(row.ordered, true); assert.equal(row.uniqueSourceIdentities, row.individualDeltas); + assert.equal(row.matchedSourceClocks, row.individualDeltas); + if (!fixed.stop) assert.equal(row.individualDeltas, fixed.events); + } + } + const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs)); + assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms"); + assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database"); + const inventory = await validateArmInventory(path.resolve(arg("original-h4d-inventory")!), { evidence: proof, owningPlaywright: owner }); + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const) + assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory"); + originalH4dConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision }; + } catch (error) { + originalH4dConfirmation.reason = error instanceof Error ? error.message : String(error); + } + } +} const metric = (item: any, key: string): number => key.split(".").reduce((value, part) => value[part], item); const measurements = names.flatMap(name => { const observed = first.baseline.cases.find((item: any) => item.name === name); @@ -323,7 +479,8 @@ const measurements = names.flatMap(name => { }); }); const result: Record = { - scope: "Compiled-app remote streaming and original local Stop performance; additional real cloud Stop qualification is required separately", + scope: "Compiled-app streaming with100 independent normal agents and the original local Stop workload; real cloud Stop is separately required", + agentTopology: first.baseline.agentTopology, transportPolicy: first.baseline.transportPolicy, additionalRealCloudStopQualificationRequired: true, admissionProtocol: first.baseline.admissionProtocol, setupActivationConcurrency: first.baseline.setupActivationConcurrency, protocolVersion: first.baseline.protocolVersion, transport: first.baseline.transport, @@ -334,11 +491,21 @@ const result: Record = { comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"), manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements, primaryLoadedWorkload: "burst512", complete: true, + ...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments, + artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4dConfirmation, + originalProtocolAcceptance: "Baseline SLA misses are retained; H5b complete observation is not an original-deadline PASS", + finalH4dFunctionalPassRequired: candidatePolicy === "final-b2" } : {}), }; if (phase === "compare") { assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report"); const noise = JSON.parse(await readFile(arg("noise-report")!, "utf8")); assert.equal(noise.phase, "aa"); assert.equal(noise.complete, true); + if (h5) { + assert.equal(noise.observationMode, H5_MODE); assert.equal(noise.workloadSha256, H5_WORKLOAD_SHA256); + assert.equal(noise.h5Assessments?.length, 5, "Fresh complete H5 A/A outcomes are required"); + for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"]) + assert.equal(noise.artifactInventories?.baseline?.[field], result.artifactInventories.baseline[field], "Baseline inventory changed after H5 A/A"); + } assert.equal(noise.protocolVersion, result.protocolVersion, "Noise report uses a different mixed protocol"); assert.equal(noise.transport, result.transport, "Noise report uses a different transport"); assert.deepEqual(noise.transportPolicy, result.transportPolicy, "Transport policy changed after A/A calibration"); @@ -362,10 +529,17 @@ if (phase === "compare") { noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins, passed: improvementRequired ? improvement : noRegression }; }); - result.acceptancePassed = result.gates.every((entry: any) => entry.passed); + result.performancePassed = result.gates.every((entry: any) => entry.passed) + && (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed)); + result.acceptancePassed = result.performancePassed && (!originalH4dConfirmation || originalH4dConfirmation.passed); } if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2)); +if (process.argv.includes("--require-performance")) { + assert.equal(h5, true, "H5b performance-only gating is explicit; default H4d keeps original deadlines"); + assert.equal(phase, "compare"); + assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose"); +} if (process.argv.includes("--require-acceptance")) { assert.equal(phase, "compare", "A/A calibrates noise; it is not performance acceptance"); assert.equal(result.acceptancePassed, true, "Browser performance acceptance failed; retain results and diagnose"); diff --git a/scripts/describe-native-ingestion-h5.mjs b/scripts/describe-native-ingestion-h5.mjs new file mode 100644 index 00000000000..b93b7f2a3e0 --- /dev/null +++ b/scripts/describe-native-ingestion-h5.mjs @@ -0,0 +1,7 @@ +import { H5_REGISTRATION, H5_WORKLOAD_SHA256 } from '../tests/e2e/fixtures/native-ingestion-h5.mjs'; +if (process.argv.length > 2) throw new Error('Dry run only: this command accepts no execution arguments'); +console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadSha256: H5_WORKLOAD_SHA256, + registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes', + 'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation', + 'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass', + 'Later H4d default-mode full functional pass on the same source, geometry and final SUT'] }, null, 2)); diff --git a/scripts/native-ingestion-h5-inventory.mjs b/scripts/native-ingestion-h5-inventory.mjs new file mode 100644 index 00000000000..e97bfc76d03 --- /dev/null +++ b/scripts/native-ingestion-h5-inventory.mjs @@ -0,0 +1,137 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; +import { gunzipSync } from 'node:zlib'; + +const SCHEMA = 'paperclip.native-ingestion.arm-inventory.v1'; +const ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src']; +const sha256 = value => createHash('sha256').update(value).digest('hex'); +const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +const revision = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value); +const object = value => value && typeof value === 'object' && !Array.isArray(value); + +async function bounded(file, maximumBytes) { + const metadata = await stat(file); + assert.ok(metadata.isFile() && metadata.size <= maximumBytes, 'Inventory evidence file is absent, nonregular or oversized'); + const bytes = await readFile(file); + assert.ok(bytes.length <= maximumBytes, 'Inventory evidence file grew beyond its bound'); + return bytes; +} + +function relativeArtifact(value) { + return typeof value === 'string' && value.length > 0 && !value.includes('\\') + && !value.includes('\0') && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value + && !value.split('/').includes('..') && ROOTS.some(root => value.startsWith(`${root}/`)); +} + +// The existing Python generator sorts Unicode code points and uses json.dumps +// with sort_keys=True, separators=(',', ':') and its default ensure_ascii=True. +function pythonOrder(left, right) { + const a = [...left], b = [...right]; + for (let index = 0; index < Math.min(a.length, b.length); index++) { + const difference = a[index].codePointAt(0) - b[index].codePointAt(0); + if (difference) return difference; + } + return a.length - b.length; +} +const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 }) + .replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`); + +/** + * Validates frozen inventory evidence and reported serving fingerprints only. + * The owner supplies this sidecar per arm; this helper never reads the SUT tree + * and does not attest that a live filesystem remained unchanged during a run. + */ +export async function validateArmInventory(sidecarFile, { evidence, owningPlaywright }) { + assert.equal(typeof sidecarFile, 'string', 'Missing arm inventory sidecar'); + assert.ok(object(evidence) && object(owningPlaywright), 'Missing serving evidence or owning Playwright report'); + const absoluteSidecar = path.resolve(sidecarFile), base = path.dirname(absoluteSidecar); + const sidecarBytes = await bounded(absoluteSidecar, 256 * 1024); + const sidecar = JSON.parse(sidecarBytes.toString('utf8')); + assert.equal(sidecar.schema, SCHEMA, 'Unsupported arm inventory sidecar schema'); + assert.ok(['B0', 'B1', 'B2'].includes(sidecar.variant), 'Unknown inventory variant'); + assert.ok(revision(sidecar.sourceRevision) && revision(sidecar.compiledBuildRevision), 'Invalid inventory source revision'); + assert.equal(sidecar.sourceRevision, evidence.sourceRevision, 'Inventory source revision differs from arm'); + assert.equal(sidecar.compiledBuildRevision, evidence.compiledBuildRevision, 'Inventory build revision differs from arm'); + assert.equal(sidecar.compiledBuildRevision, sidecar.sourceRevision, 'Inventory was built from a different source revision'); + assert.ok(digest(sidecar.playwrightReportSha256) && digest(owningPlaywright.reportSha256), 'Missing owning Playwright report hash'); + assert.equal(sidecar.playwrightReportSha256, owningPlaywright.reportSha256, 'Inventory belongs to another owning Playwright report'); + const resolve = name => { + assert.ok(typeof sidecar[name] === 'string' && sidecar[name].length > 0, `Missing inventory ${name}`); + return path.resolve(base, sidecar[name]); + }; + const readHashed = async (name, hashName, maximumBytes) => { + assert.ok(digest(sidecar[hashName]), `Invalid inventory ${hashName}`); + const file = resolve(name), bytes = await bounded(file, maximumBytes); + assert.equal(sha256(bytes), sidecar[hashName], `Inventory ${name} bytes changed`); + return { file, bytes }; + }; + const manifestSource = await readHashed('manifestFile', 'manifestFileSha256', 64 * 1024 * 1024); + const manifest = JSON.parse(gunzipSync(manifestSource.bytes, { maxOutputLength: 128 * 1024 * 1024 }).toString('utf8')); + assert.deepEqual(Object.keys(manifest).sort(), ['aggregateSha256', 'compiledBuildRevision', 'fileCount', 'files', 'revision', 'roots', 'variant']); + assert.equal(manifest.variant, sidecar.variant, 'Inventory manifest variant differs from sidecar'); + assert.equal(manifest.revision, sidecar.sourceRevision, 'Inventory manifest source revision differs from arm'); + assert.equal(manifest.compiledBuildRevision, sidecar.compiledBuildRevision, 'Inventory manifest build revision differs from arm'); + assert.deepEqual(manifest.roots, ROOTS, 'Full registered inventory roots are required'); + assert.ok(Number.isSafeInteger(manifest.fileCount) && manifest.fileCount > 0 && manifest.fileCount <= 500000, 'Invalid inventory file count'); + assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount, 'Inventory file count is incomplete'); + assert.ok(digest(manifest.aggregateSha256) && digest(sidecar.aggregateSha256), 'Invalid inventory aggregate hash'); + const files = new Map(); + for (const row of manifest.files) { + assert.ok(object(row), 'Invalid inventory file row'); + assert.deepEqual(Object.keys(row).sort(), ['bytes', 'path', 'sha256']); + assert.ok(relativeArtifact(row.path), 'Inventory file escapes registered roots'); + assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0 && digest(row.sha256), 'Invalid inventory file descriptor'); + assert.equal(files.has(row.path), false, 'Duplicate inventory file path'); + files.set(row.path, row); + } + const sorted = [...manifest.files].sort((left, right) => pythonOrder(left.path, right.path)); + assert.deepEqual(manifest.files.map(row => row.path), sorted.map(row => row.path), 'Inventory file order differs from generator'); + for (const root of ROOTS) assert.ok(manifest.files.some(row => row.path.startsWith(`${root}/`)), 'Inventory omitted a registered root'); + const aggregate = sha256(Buffer.from(`${sorted.map(pythonRow).join('\n')}\n`, 'utf8')); + assert.equal(aggregate, manifest.aggregateSha256, 'Inventory aggregate does not match exact file descriptors'); + assert.equal(aggregate, sidecar.aggregateSha256, 'Inventory aggregate differs from sidecar'); + assert.ok(digest(evidence.serverEntrySha256), 'Missing serving server entry hash'); + assert.equal(files.get('server/dist/index.js')?.sha256, evidence.serverEntrySha256, 'Serving server entry differs from full inventory'); + assert.ok(object(evidence.compiledHotPathHashes) && Object.keys(evidence.compiledHotPathHashes).length > 0, 'Missing serving hot-path fingerprints'); + for (const [file, hash] of Object.entries(evidence.compiledHotPathHashes)) { + assert.ok(relativeArtifact(file) && digest(hash), 'Invalid serving hot-path fingerprint'); + assert.equal(files.get(file)?.sha256, hash, 'Serving hot path differs from full inventory'); + } + const summarySource = await readHashed('inventorySummaryFile', 'inventorySummarySha256', 1024 * 1024); + const summary = JSON.parse(summarySource.bytes.toString('utf8')); + const summaryRow = summary.inventories?.[sidecar.variant]; + assert.ok(object(summaryRow), 'Inventory summary omitted this variant'); + assert.ok(typeof summaryRow.manifest === 'string' && path.basename(summaryRow.manifest) === summaryRow.manifest + && summaryRow.manifest.endsWith('.json.gz'), 'Inventory summary manifest reference is invalid'); + assert.equal(summaryRow.manifestSha256, sidecar.manifestFileSha256, 'Inventory summary points to different manifest bytes'); + assert.equal(summaryRow.aggregateSha256, aggregate, 'Inventory summary aggregate differs'); + assert.equal(summaryRow.fileCount, manifest.fileCount, 'Inventory summary file count differs'); + const receiptSource = await readHashed('qualifiedReceiptFile', 'qualifiedReceiptSha256', 4 * 1024 * 1024); + const receipt = JSON.parse(receiptSource.bytes.toString('utf8')); + assert.ok(Array.isArray(receipt.results), 'Qualified build receipt has no results'); + const qualified = receipt.results.filter(row => row.variant === sidecar.variant); + assert.equal(qualified.length, 1, 'Qualified build receipt must identify this variant exactly once'); + assert.equal(qualified[0].revision, sidecar.sourceRevision, 'Qualified build receipt source revision differs'); + assert.equal(qualified[0].compiledBuildRevision, sidecar.compiledBuildRevision, 'Qualified build receipt build revision differs'); + assert.equal(qualified[0].sourceClean, true, 'Qualified build source was not clean'); + assert.ok(typeof receipt.artifactInventorySummary === 'string' && receipt.artifactInventorySummary.length > 0, + 'Qualified build receipt omitted its inventory summary reference'); + assert.ok(revision(receipt.rustSourceTree), 'Qualified build receipt omitted Rust input identity'); + assert.ok(Array.isArray(receipt.nativeStageHashes), 'Qualified build receipt omitted native stage hashes'); + const native = receipt.nativeStageHashes.filter(row => row.variant === sidecar.variant); + assert.equal(native.length, 1, 'Qualified build receipt must identify this native stage exactly once'); + assert.ok(digest(native[0].binarySha256) && digest(receipt.binarySha256), 'Qualified native binary hash is invalid'); + assert.equal(native[0].binarySha256, receipt.binarySha256, 'Qualified native binary hashes disagree'); + assert.equal(native[0].binarySha256, evidence.remote?.runner?.sha256, 'Serving native binary differs from qualified build'); + return { schema: SCHEMA, sidecarFile: absoluteSidecar, sidecarFileSha256: sha256(sidecarBytes), variant: sidecar.variant, + revision: manifest.revision, compiledBuildRevision: manifest.compiledBuildRevision, + manifestFile: manifestSource.file, manifestFileSha256: sidecar.manifestFileSha256, + aggregateSha256: aggregate, fileCount: manifest.fileCount, roots: manifest.roots, + inventorySummaryFile: summarySource.file, inventorySummarySha256: sidecar.inventorySummarySha256, + qualifiedReceiptFile: receiptSource.file, qualifiedReceiptSha256: sidecar.qualifiedReceiptSha256, + binarySha256: native[0].binarySha256, rustSourceTree: receipt.rustSourceTree, + playwrightReportSha256: owningPlaywright.reportSha256, + scope: 'Exact frozen manifest/summary/build-receipt bytes for all registered roots, with reported server/hot-path/native fingerprints checked; no live SUT filesystem attestation' }; +} diff --git a/scripts/native-ingestion-h5-inventory.test.mjs b/scripts/native-ingestion-h5-inventory.test.mjs new file mode 100644 index 00000000000..25323d07894 --- /dev/null +++ b/scripts/native-ingestion-h5-inventory.test.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { gzipSync } from 'node:zlib'; +import { validateArmInventory } from './native-ingestion-h5-inventory.mjs'; + +const hash = value => createHash('sha256').update(value).digest('hex'); +const sourceRevision = '1'.repeat(40), reportSha256 = hash('owning-playwright'), binarySha256 = hash('runner'); +const roots = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src']; +const descriptors = () => [ + 'packages/db/src/client.ts', 'packages/shared/src/index.ts', 'server/dist/hot-a.js', + 'server/dist/hot-b.js', 'server/dist/hot-c.js', 'server/dist/index.js', 'ui/dist/index.html', +].map(file => ({ bytes: 10, path: file, sha256: hash(file) })); +const aggregate = files => hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join('\n') + '\n'); + +async function fixture(directory, options = {}) { + const revision = options.revision ?? sourceRevision; + const files = descriptors(), manifest = { variant: 'B2', revision, compiledBuildRevision: revision, + roots, fileCount: files.length, aggregateSha256: aggregate(files), files }; + options.manifest?.(manifest); + const manifestBytes = gzipSync(Buffer.from(JSON.stringify(manifest))); + const summary = { capturedAt: '2026-11-03T00:00:00.000Z', inventories: { B2: { + manifest: 'B2-artifact-inventory.json.gz', manifestSha256: hash(manifestBytes), fileCount: manifest.fileCount, + aggregateSha256: manifest.aggregateSha256 } } }; + options.summary?.(summary); + const receipt = { completedAt: '2026-11-03T00:00:01.000Z', + results: [{ variant: 'B2', revision, compiledBuildRevision: revision, sourceClean: true }], + rustSourceTree: '2'.repeat(40), binarySha256, nativeStageHashes: [{ variant: 'B2', binarySha256 }], + artifactInventorySummary: '/owned/original/artifact-inventory-summary.json' }; + options.receipt?.(receipt); + const summaryBytes = Buffer.from(JSON.stringify(summary)), receiptBytes = Buffer.from(JSON.stringify(receipt)); + const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant: 'B2', sourceRevision: revision, + compiledBuildRevision: revision, playwrightReportSha256: reportSha256, + manifestFile: 'B2-artifact-inventory.json.gz', manifestFileSha256: hash(manifestBytes), + aggregateSha256: manifest.aggregateSha256, + qualifiedReceiptFile: 'qualified.json', qualifiedReceiptSha256: hash(receiptBytes), + inventorySummaryFile: 'artifact-inventory-summary.json', inventorySummarySha256: hash(summaryBytes) }; + options.sidecar?.(sidecar); + const evidence = { sourceRevision: revision, compiledBuildRevision: revision, + serverEntrySha256: hash('server/dist/index.js'), remote: { runner: { sha256: binarySha256 } }, + compiledHotPathHashes: Object.fromEntries(['server/dist/hot-a.js', 'server/dist/hot-b.js', 'server/dist/hot-c.js'].map(file => [file, hash(file)])) }; + options.evidence?.(evidence); + await Promise.all([ + writeFile(path.join(directory, sidecar.manifestFile), manifestBytes), + writeFile(path.join(directory, sidecar.inventorySummaryFile), summaryBytes), + writeFile(path.join(directory, sidecar.qualifiedReceiptFile), receiptBytes), + writeFile(path.join(directory, 'arm-inventory.json'), JSON.stringify(sidecar)), + ]); + return { sidecarFile: path.join(directory, 'arm-inventory.json'), sidecar, evidence, + owningPlaywright: { reportSha256 }, manifest, summary, receipt }; +} + +async function withFixture(options, work) { + const directory = await mkdtemp(path.join(os.tmpdir(), 'h5-inventory-')); + try { await work(await fixture(directory, options), directory); } + finally { await rm(directory, { recursive: true, force: true }); } +} + +test('full frozen inventory, qualified native build and owning report bind one arm', async () => { + await withFixture({}, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.manifestFileSha256, value.sidecar.manifestFileSha256); + assert.equal(proof.aggregateSha256, aggregate(descriptors())); + assert.equal(proof.fileCount, descriptors().length); + assert.equal(proof.qualifiedReceiptSha256, value.sidecar.qualifiedReceiptSha256); + assert.equal(proof.playwrightReportSha256, reportSha256); + assert.equal(proof.binarySha256, binarySha256); + assert.deepEqual(proof.roots, roots); + }); +}); + +test('exact compressed manifest bytes and build-receipt bytes cannot be swapped', async () => { + for (const field of ['manifestFile', 'qualifiedReceiptFile']) await withFixture({}, async (value, directory) => { + const file = path.join(directory, value.sidecar[field]); + await writeFile(file, Buffer.concat([await readFile(file), Buffer.from(' ')])); + await assert.rejects(validateArmInventory(value.sidecarFile, value), /bytes changed/); + }); +}); + +test('source, compiled build and owning report must match the arm', async () => { + const cases = [ + [{ sidecar: row => { row.sourceRevision = '3'.repeat(40); } }, /source revision differs/], + [{ sidecar: row => { row.compiledBuildRevision = '3'.repeat(40); } }, /build revision differs/], + [{ sidecar: row => { row.playwrightReportSha256 = hash('another-report'); } }, /another owning Playwright/], + [{ manifest: row => { row.revision = '3'.repeat(40); } }, /manifest source revision/], + [{ receipt: row => { row.results[0].revision = '3'.repeat(40); } }, /receipt source revision/], + [{ receipt: row => { row.results[0].compiledBuildRevision = '3'.repeat(40); } }, /receipt build revision/], + [{ receipt: row => { row.results[0].sourceClean = false; } }, /source was not clean/], + ]; + for (const [options, message] of cases) await withFixture(options, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); +}); + +test('file descriptors must be complete, bounded, distinct and match the Python aggregate', async () => { + const cases = [ + [row => { row.fileCount--; }, /file count is incomplete/], + [row => { row.aggregateSha256 = hash('wrong aggregate'); }, /aggregate does not match/], + [row => { row.files[0].bytes = -1; }, /file descriptor/], + [row => { row.files[0].path = '../outside'; }, /escapes registered roots/], + [row => { row.files[1].path = row.files[0].path; }, /Duplicate/], + [row => { row.files.reverse(); }, /order differs/], + [row => { row.files.pop(); row.fileCount--; row.aggregateSha256 = aggregate(row.files); }, /omitted a registered root/], + [row => { row.roots = ['server/dist']; }, /registered inventory roots/], + ]; + for (const [change, message] of cases) await withFixture({ manifest: change }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); +}); + +test('summary identifies exact gzip, count and aggregate for the selected variant', async () => { + for (const [key, value] of [['manifestSha256', hash('wrong')], ['fileCount', 1], ['aggregateSha256', hash('wrong')]]) { + await withFixture({ summary: row => { row.inventories.B2[key] = value; } }, async fixture => { + await assert.rejects(validateArmInventory(fixture.sidecarFile, fixture), /Inventory summary/); + }); + } +}); + +test('serving entry, every hot path and native binary must match full evidence', async () => { + const cases = [ + [row => { row.serverEntrySha256 = hash('wrong-entry'); }, /server entry differs/], + [row => { row.compiledHotPathHashes['server/dist/hot-c.js'] = hash('wrong-hot-path'); }, /hot path differs/], + [row => { row.remote.runner.sha256 = hash('wrong-native'); }, /native binary differs/], + ]; + for (const [change, message] of cases) await withFixture({ evidence: change }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), message); + }); + await withFixture({ receipt: row => { row.nativeStageHashes.push({ ...row.nativeStageHashes[0] }); } }, async value => { + await assert.rejects(validateArmInventory(value.sidecarFile, value), /native stage exactly once/); + }); +}); + +test('a future candidate revision is accepted only with matching complete provenance', async () => { + await withFixture({ revision: 'f'.repeat(40) }, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.revision, 'f'.repeat(40)); + assert.equal(proof.compiledBuildRevision, 'f'.repeat(40)); + }); +}); + +test('aggregate follows the existing Python ensure_ascii convention for non-ASCII paths', async () => { + // Independently computed with the retained artifact-inventory-ce33.py + // json.dumps(sort_keys=True, separators=(',', ':')) canonical convention. + const expected = '6f7bf8cab07e2aaf8faaa95a8bdb565c588670e4e3e3580acdba1f93154fedde'; + await withFixture({ manifest: row => { + row.files.at(-1).path = 'ui/dist/é😀\u007f.html'; row.aggregateSha256 = expected; + } }, async value => { + const proof = await validateArmInventory(value.sidecarFile, value); + assert.equal(proof.aggregateSha256, expected); + }); +}); diff --git a/scripts/native-ingestion-h5.test.mjs b/scripts/native-ingestion-h5.test.mjs new file mode 100644 index 00000000000..9e074d97cd1 --- /dev/null +++ b/scripts/native-ingestion-h5.test.mjs @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createRequire } from 'node:module'; +import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate, + H5_LIMITS, ORIGINAL_BUDGETS, prepareParticipantAgents } from '../tests/e2e/fixtures/native-ingestion-h5.mjs'; + +const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`), + { matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } }); + +test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => { + const require = createRequire(import.meta.url), { expect } = require('@playwright/test'); + assert.equal(require('@playwright/test/package.json').version, '1.62.1'); + await assert.rejects(expect.poll(async () => 0, { timeout: 15, intervals: [5] }).toBeGreaterThanOrEqual(1), error => { + assert.equal(error.matcherResult?.name, 'toBeGreaterThanOrEqual'); + assert.equal(error.matcherResult.pass, false); + assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate')); + return true; + }); + // This checks the real library's format; it does not execute a120s wait or + // claim runtime qualification of H5's extended application path. +}); + +test('real frozen Playwright can report its configured timeout before the nominal deadline', async context => { + const require = createRequire(import.meta.url), { expect } = require('@playwright/test'); + assert.equal(require('@playwright/test/package.json').version, '1.62.1'); + const timeoutMs = 1200, intervalMs = 1500, startedAtMs = Date.now(), started = performance.now(); + let observations = 0, failure; + try { + await expect.poll(async () => { observations++; return 0; }, { timeout: timeoutMs, intervals: [intervalMs] }).toBeGreaterThanOrEqual(1); + } catch (error) { failure = error; } + const assessedAtMs = Date.now(), elapsedMs = performance.now() - started; + assert.equal(failure?.matcherResult?.name, 'toBeGreaterThanOrEqual'); + assert.equal(failure.matcherResult.pass, false); + assert.ok(failure.message.includes(`Timeout ${timeoutMs}ms exceeded while waiting on the predicate`)); + assert.equal(observations, 1); + assert.ok(assessedAtMs < startedAtMs + timeoutMs, 'The real poll ends before its nominal deadline when its next interval would cross it'); + context.diagnostic(JSON.stringify({ scope: 'Small real-library poll only; no app or120s runtime qualification', + playwrightVersion: '1.62.1', timeoutMs, intervalMs, startedAtMs, nominalDeadlineMs: startedAtMs + timeoutMs, + assessedAtMs, elapsedMs, observations, matcherName: failure.matcherResult.name, matcherPassed: failure.matcherResult.pass })); +}); + +test('H5b requires its explicit mode and leaves default H4d selected', () => { + assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true); + assert.throws(() => h5Enabled('almost-h5')); + assert.throws(() => h5Enabled('h5-observe-original-sla'), 'Old H5a observations use a different geometry'); +}); + +for (const fail of [false, true]) test(`participant setup uses at most8 workers, preserves index order and settles all attempts${fail ? ' after a failure' : ''}`, async () => { + const count = 20, pending = [], records = []; let active = 0, maximum = 0, finished = 0; + const completion = prepareParticipantAgents({ count, onOutcome: value => records.push(structuredClone(value)), + createAndReady: async index => { + active++; maximum = Math.max(maximum, active); + await new Promise(resolve => pending.push(resolve)); active--; finished++; + if (fail && index === 3) throw new Error('actual agent readiness failed'); + return { id: `agent-${index}` }; + } }).then(value => ({ value }), error => ({ error })); + while (finished < count) { + await new Promise(resolve => setImmediate(resolve)); + assert.ok(active <= 8); pending.splice(0).reverse().forEach(resolve => resolve()); + } + const result = await completion, last = records.at(-1); + assert.equal(maximum, 8); assert.equal(active, 0); assert.equal(last.attempts.length, count); + assert.ok(last.attempts.every(attempt => attempt.status !== 'pending')); + assert.equal(last.complete, !fail); + if (fail) { assert.match(result.error.message, /setup failed/); assert.equal(last.attempts.filter(value => value.status === 'rejected').length, 1); } + else assert.deepEqual(result.value.map(agent => agent.id), Array.from({ length: count }, (_, index) => `agent-${index}`)); +}); + +test('first successful observation uses only the original wait with no collection delay', async () => { + const calls = []; + const value = await observeStreaming({ readCount: async () => 16, target: 16, now: () => 1000, + poll: async (read, milliseconds, target) => { calls.push(milliseconds); assert.ok(await read() >= target); } }); + assert.deepEqual(calls, [120000]); assert.equal(value.original.status, 'passed'); + assert.equal(value.observation.complete, true); assert.equal(value.observation.completedAtMs, 1000); +}); + +test('known120s miss assessed before its nominal deadline stays failed and uses only the remaining600s observation', async () => { + let clock = 1000, count = 30; const calls = [], records = []; + const value = await observeStreaming({ readCount: async () => count, target: 512, now: () => clock, + onOutcome: value => records.push(structuredClone(value)), + poll: async (read, milliseconds) => { + calls.push(milliseconds); await read(); + if (calls.length === 1) { clock += milliseconds - 750; throw timeout(milliseconds); } + count = 512; clock += 90000; await read(); + } }); + assert.deepEqual(calls, [120000, 480750]); + assert.equal(value.original.status, 'failed'); assert.equal(value.original.lastObservedCount, 30); + assert.equal(value.original.lastObservedAtMs, 1000); assert.equal(value.original.deadlineMs, 121000); + assert.equal(value.original.assessedAtMs, 120250); assert.ok(value.original.assessedAtMs < value.original.deadlineMs); + assert.equal(value.observation.completedAtMs, 210250); assert.equal(records[0].observation.complete, false); + assert.equal(value.original.reason, 'original_streaming_deadline'); +}); + +test('page read failures and unknown poll failures never become an expected SLA miss', async () => { + const failure = new Error('page detached'); + await assert.rejects(observeStreaming({ readCount: async () => { throw failure; }, target: 1, + poll: async read => { try { await read(); } catch {} throw timeout(120000); } }), error => error === failure); + await assert.rejects(observeStreaming({ readCount: async () => 0, target: 1, + poll: async () => { throw new Error('unrelated assertion'); } }), /Only the original streaming matcher deadline/); +}); + +test('observation ceiling leaves a censored failure, never a substituted latency', async () => { + let clock = 1000, calls = 0; const records = []; + await assert.rejects(observeStreaming({ readCount: async () => 30, target: 512, now: () => clock, + onOutcome: value => records.push(structuredClone(value)), + poll: async (read, milliseconds) => { await read(); calls++; clock += milliseconds; throw timeout(milliseconds); } }), /Timeout/); + assert.equal(calls, 2); assert.equal(clock, 601000); + assert.equal(records.at(-1).observation.complete, false); assert.equal(records.at(-1).observation.censored, true); + assert.equal(records.at(-1).observation.completedAtMs, null); +}); + +test('original terminal is not reached after120s miss; observational180s is distinct', () => { + const terminal = { complete: true, startedAtMs: 211000, deadlineMs: 391000, assessedAtMs: 212000 }; + assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), { + status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null }); + assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed'); + assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs); +}); + +test('cursor inherits only paired exact known misses; final candidate inherits none', () => { + const base = { misses: ['burst512:streaming120'] }; + requireH5Candidate('cursor-nonregression', base, base); + const slowInvocation = { misses: ['invocation:observed-total-duration900'] }; + requireH5Candidate('cursor-nonregression', slowInvocation, slowInvocation); + requireH5Candidate('final-b2', base, { misses: [] }); + for (const candidate of [{ misses: ['paced:streaming120'] }, { misses: ['invocation:observed-total-duration900'] }, + { misses: ['operator-stop:streaming120'] }]) assert.throws(() => requireH5Candidate('cursor-nonregression', base, candidate)); + assert.throws(() => requireH5Candidate('final-b2', base, base), /every original SLA/); + assert.throws(() => requireH5Candidate('final-b2', slowInvocation, slowInvocation), /every original SLA/); +}); diff --git a/scripts/native-ingestion-performance-gates.test.mjs b/scripts/native-ingestion-performance-gates.test.mjs index 21753ab312c..4b7ce897e3f 100644 --- a/scripts/native-ingestion-performance-gates.test.mjs +++ b/scripts/native-ingestion-performance-gates.test.mjs @@ -1,10 +1,13 @@ import assert from "node:assert/strict"; +import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, + originalTerminal, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; +import { gzipSync } from "node:zlib"; import { evaluateServiceRepeatability, noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs"; test("four pairs must individually improve beyond noise", () => { @@ -77,17 +80,23 @@ const workloads = [ function browserReport(revision, latency) { const companyId = "00000000-0000-4000-8000-000000000001", environmentId = "00000000-0000-4000-8000-000000000002"; const localEnvironmentId = "00000000-0000-4000-8000-000000000003", controlAgentId = "00000000-0000-4000-8000-000000000004"; - const remoteAgentId = "00000000-0000-4000-8000-000000000005"; const id = (kind, caseIndex, index) => `00000000-0000-4000-8000-${(kind * 100000 + caseIndex * 1000 + index).toString(16).padStart(12, "0")}`; + const remoteAgentIds = Array.from({ length: 100 }, (_, index) => id(9, 0, index)); const transportPolicy = { streaming: "sandbox-remote", operatorStop: "local", realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }; - return { protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport: "sandbox-remote-with-local-stop", transportPolicy, companyId, + return { protocolVersion: H4D_PROTOCOL, transport: "sandbox-remote-with-local-stop", transportPolicy, companyId, + bodyStartedAtMs: 1, agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: 100 }, + harnessProvenance: { sourceRevision: "b".repeat(40), sourceStatus: "", driverLockSha256: "8".repeat(64), + sourceHashes: { "native-event-ingestion.journey.ts": "9".repeat(64), "fixtures/native-ingestion-h5.mjs": "a".repeat(64) } }, resourceBudgets: { providerReadinessMs: 300000, fullInvocationMs: 900000, streamingMs: 120000, terminalDrainageMs: 180000, stopAcknowledgementMs: 30000, remoteLeaseCleanupMs: 30000 }, local: { transport: "local", environmentId: localEnvironmentId, controlAgentId, selection: "Normal local environment + separate control agent defaultEnvironmentId; existing test-only Codex leaf command fixture", commandSha256: "6".repeat(64), loaderSha256: "7".repeat(64) }, - remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId, agentIds: [remoteAgentId], + remote: { transport: "sandbox-remote", revision, buildRevision: revision, environmentId, agentIds: remoteAgentIds, + agentSetup: { count: 100, concurrency: 8, complete: true, startedAtMs: 2, settledAtMs: 40, + attempts: remoteAgentIds.map((agentId, index) => ({ index, agentId, status: "fulfilled", + startedAtMs: 3 + Math.floor(index / 8) * 2, settledAtMs: 4 + Math.floor(index / 8) * 2 })) }, runnerPublicUrl: "wss://127.0.0.1:32100", selection: "Normal plugin install + sandbox environment + agent defaultEnvironmentId; no runtime resolver override", plugin: { pluginKey: "paperclip.fake-sandbox-provider", assetHashes: Object.fromEntries(["worker.js", "plugin.js", "manifest.js", "lease-cleanup.js"].map(file => [file, "5".repeat(64)])) }, @@ -104,7 +113,7 @@ function browserReport(revision, latency) { observedReadOnly: "on", observedIsolation: "repeatable read", kind: stop ? "host-file-journal" : "database-controller-checkpoint" }; const rows = Array.from({ length: workload.runs }, (_, index) => ({ run_id: id(1, caseIndex, index), issue_id: id(2, caseIndex, index), company_id: companyId, - agent_id: stop ? controlAgentId : remoteAgentId, + agent_id: stop ? controlAgentId : remoteAgentIds[index], lease_id: id(3, caseIndex, index), environment_id: actualEnvironmentId, runtime_mode: "native", provider: stop ? "local" : "fake-plugin", provider_lease_id: stop ? null : `fake-plugin://${id(1, caseIndex, index)}/fixture`, checkpoint_part: stop ? null : "header", checkpoint_header_bytes: stop ? null : 256, @@ -161,6 +170,224 @@ function browserReport(revision, latency) { }) }; } +function h5Report(revision, loadedLatency) { + const report = browserReport(({ ...H5_BASELINE_REVISIONS, B2: "c".repeat(40) })[revision], 1000); + Object.assign(report, { protocolVersion: H5_PROTOCOL, observationMode: H5_MODE, + workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, observationComplete: true, + actualPlaywrightTimeoutMs: H5_LIMITS.fullInvocationMs, measurementSha256: "h5-synthetic-fixed-fixture", + serverEntrySha256: "e".repeat(64), compiledHotPathHashes: { + "server/dist/services/heartbeat-run-events.js": "f".repeat(64), + "server/dist/services/native-runtime/paperclip-control-plane-port.js": "1".repeat(64), + "server/dist/vendor/paperclip-runner/native-session-runtime.js": "2".repeat(64) } }); + for (const item of report.cases) { + const latency = item.name === "burst512" ? loadedLatency : 1000; + item.settledAtMs = item.epochMs + latency; item.drainThroughTerminalMs = latency; + item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency }; + item.intendedToVisible = { p95Ms: latency }; + if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1; + item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs })); + const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs; + const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null, + startedAtMs: item.epochMs, deadlineMs: item.epochMs + 120000, assessedAtMs, + target: item.stop ? 3 : item.events, lastObservedCount: missed ? 30 : item.visible.length, + lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs, + error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null }, + observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } }; + const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000, + assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs, + lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false }; + item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation), + visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length, + independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10, + deltaCount: item.runs * (item.stop ? 3 : item.events), resultAcceptedCount: item.stop ? 0 : item.runs, clockStreamsClosed: true } }; + item.terminalStatuses = item.runIds.map(runId => ({ runId, runtimeMode: "native", status: item.stop ? "cancelled" : "succeeded" })); + for (const row of item.durable) row.resultAcceptedEvents = item.stop ? 0 : 1; + } + return report; +} + +test("H5 assessor retains an original SLA failure assessed before its nominal streaming deadline", () => { + const report = h5Report("B0", 180000), streaming = report.cases[3].h5.streaming.original; + streaming.assessedAtMs = streaming.deadlineMs - 750; + streaming.lastObservedAtMs = streaming.assessedAtMs - 1; + const outcome = assessH5Arm(report, { status: "passed", retry: 0, errors: [], timeoutMs: H5_LIMITS.fullInvocationMs, + startedAt: "2026-11-03T00:00:00.000Z", durationMs: 500000 }); + assert.equal(outcome.originalSlaPassed, false); + assert.deepEqual(outcome.misses, ["burst512:streaming120"]); + assert.throws(() => requireH5Candidate("final-b2", outcome, outcome), /every original SLA/); +}); + +test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires complete owning evidence", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-")); + const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url); + const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; }; + const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" }); + const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map(); + async function inventory(report, playwrightFile, name) { + const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision]; + if (!inventories.has(variant)) { + const roots = ["server/dist", "ui/dist", "packages/db/src", "packages/shared/src"]; + const files = Object.entries({ ...report.compiledHotPathHashes, "server/dist/index.js": report.serverEntrySha256, + "packages/db/src/client.ts": "3".repeat(64), "packages/shared/src/index.ts": "4".repeat(64), "ui/dist/index.html": "5".repeat(64) }) + .map(([path, sha256]) => ({ bytes: 10, path, sha256 })).sort((left, right) => left.path.localeCompare(right.path, "en")); + const aggregateSha256 = hash(files.map(row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })).join("\n") + "\n"); + const manifestBytes = gzipSync(Buffer.from(JSON.stringify({ variant, revision: report.sourceRevision, + compiledBuildRevision: report.compiledBuildRevision, roots, fileCount: files.length, files, aggregateSha256 }))); + const manifestFile = path.join(directory, `${variant}-artifact-inventory.json.gz`); await writeFile(manifestFile, manifestBytes); + const manifestFileSha256 = hash(manifestBytes); + const inventorySummaryFile = await save(`${variant}-summary.json`, { inventories: { [variant]: { + manifest: path.basename(manifestFile), manifestSha256: manifestFileSha256, aggregateSha256, fileCount: files.length } } }); + const qualifiedReceiptFile = await save(`${variant}-qualified.json`, { results: [{ variant, revision: report.sourceRevision, + compiledBuildRevision: report.compiledBuildRevision, sourceClean: true }], rustSourceTree: "9".repeat(40), + binarySha256: report.remote.runner.sha256, nativeStageHashes: [{ variant, binarySha256: report.remote.runner.sha256 }], + artifactInventorySummary: inventorySummaryFile }); + inventories.set(variant, { manifestFile, manifestFileSha256, aggregateSha256, inventorySummaryFile, + inventorySummarySha256: hash(await readFile(inventorySummaryFile)), qualifiedReceiptFile, + qualifiedReceiptSha256: hash(await readFile(qualifiedReceiptFile)) }); + } + return save(`${name}-inventory.json`, { schema: "paperclip.native-ingestion.arm-inventory.v1", variant, + sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision, + playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) }); + } + async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) { + const entries = []; + for (let index = 0; index < 5; index++) { + const entry = {}; + for (const arm of ["baseline", "candidate"]) { + const report = h5Report(arm === "baseline" || sameRevision ? baselineRevision : candidateRevision, arm === "baseline" ? 180000 : candidateLatency); + const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0, + duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] }; + if (index === 0 && arm === "candidate") mutate?.(report, owner); + owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json", + body: Buffer.from(JSON.stringify(report)).toString("base64") }); + entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report); + entry[`${arm}PlaywrightFile`] = await save(`${prefix}-${index}-${arm}-playwright.json`, { suites: [{ tests: [owner] }] }); + entry[`${arm}InventoryFile`] = await inventory(report, entry[`${arm}PlaywrightFile`], `${prefix}-${index}-${arm}`); + } + entries.push(entry); + } + return save(`${prefix}-pairs.json`, entries); + } + try { + const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-")); + proof.protocolVersion = H4D_PROTOCOL; + proof.measurementSha256 = "h5-synthetic-fixed-fixture"; + // H4d defaults use the same frozen source/geometry as the observation arms. + for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field]; + for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; } + const originalProof = await save("original-h4d-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{ + status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{ + name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] }); + const originalInventory = await inventory(proof, originalProof, "original-h4c"); + const finalProofArgs = [`--original-h4d-proof=${originalProof}`, `--original-h4d-inventory=${originalInventory}`]; + const noise = path.join(directory, "aa.json"); + const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]); + assert.equal(aa.status, 0, aa.stderr); + const aaReport = JSON.parse(await readFile(noise, "utf8")); + assert.equal(aaReport.h5Assessments.length, 5); + assert.deepEqual(aaReport.h5Assessments[0].baseline.misses, ["burst512:streaming120"]); + const cursor = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor", 180000, { candidateRevision: "B1" })}`, `--noise-report=${noise}`, "--require-acceptance"]); + assert.equal(cursor.status, 0, cursor.stderr); + const cursor900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor-matched900", 180000, { candidateRevision: "B1", duration: 950000 })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.equal(cursor900.status, 0, cursor900.stderr); + const added900 = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", + `--pairs=${await manifest("cursor-added900", 180000, { candidateRevision: "B1", mutate: (_report, owner) => { owner.results[0].duration = 950000; } })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(added900.status, 0, "Cursor candidate cannot add an unmatched900 miss"); + const goodPairs = await manifest("good", 1000); + const good = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, ...finalProofArgs, "--require-acceptance"]); + assert.equal(good.status, 0, good.stderr); + const wrongPolicy = run(["--expect=nonregression", "--candidate-policy=cursor-nonregression", `--pairs=${goodPairs}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(wrongPolicy.status, 0); assert.match(wrongPolicy.stderr, /Cursor policy requires B0→B1/); + const b1Noise = path.join(directory, "aa-b1.json"); + const b1AA = run(["--phase=aa", `--pairs=${await manifest("aa-b1", 181000, { sameRevision: true, baselineRevision: "B1" })}`, `--output=${b1Noise}`]); + assert.equal(b1AA.status, 0, b1AA.stderr); + const incremental = run(["--candidate-policy=final-b2", `--pairs=${await manifest("b1-b2", 1000, { baselineRevision: "B1" })}`, + `--noise-report=${b1Noise}`, ...finalProofArgs, "--require-acceptance"]); + assert.equal(incremental.status, 0, incremental.stderr); + const pendingFile = path.join(directory, "pending-h4c.json"); + const pending = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, `--output=${pendingFile}`, "--require-acceptance"]); + assert.notEqual(pending.status, 0); + const pendingReport = JSON.parse(await readFile(pendingFile, "utf8")); + assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false); + const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]); + assert.equal(performanceOnly.status, 0, performanceOnly.stderr); + for (const [name, change, message] of [ + ["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/], + ["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/], + ["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /H4d default deadlines/], + ["old-h4c-proof", value => { value.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream"; + value.harnessProvenance.sourceRevision = "7d7e7a88d562a95a1d407b76474763b666c181f5"; }, /Old H4c geometry/], + ["h4d-wrong-harness", value => { value.harnessProvenance.sourceRevision = "d".repeat(40); }, /same frozen harness source/], + ["h4d-wrong-measurement", value => { value.measurementSha256 = "old-h5a"; }, /same measured harness bytes/], + ["h4d-shared-agent", value => { value.cases[3].controllerAuthority.rows[1].agent_id = value.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/], + ["h4d-missing-stop-proof", value => { delete value.cases[5].localStopProof; }, /physical Stop/], + ["h4d-provider-still-alive", value => { value.cases[5].localStopProof.processes.providerAbsent = false; }, /provider remains alive/], + ["h4d-pending-local-lease", value => { value.cases[5].localStopProof.lease.status = "active"; }, /lease cleanup remains unsettled/], + ["h4d-wrong-local-lease", value => { value.cases[5].localStopProof.lease.id = value.cases[3].controllerAuthority.rows[0].lease_id; }, /strictly equal/], + ["h4d-ack-timing-mismatch", value => { value.cases[5].stopAcknowledgedAtMs++; }, /acknowledgement timing boundary/], + ["h4d-missing-remote-leases", value => { delete value.cases[3].remoteLeases; }, /post-timing lease cleanup proof/], + ["h4d-pending-remote-lease", value => { value.cases[3].remoteLeases[0].cleanupStatus = "pending"; }, /lease cleanup remains unsettled/], + ["h4d-wrong-remote-lease", value => { value.cases[3].remoteLeases[0].id = value.cases[5].controllerAuthority.rows[0].lease_id; }, /strictly equal/], + ["h4d-premature-remote-cleanup-proof", value => { value.cases[3].remoteLeasesCheckedAtMs = value.cases[3].settledAtMs - 1; }, /leases after recorded timing/], + ["h4d-wrong-authority-company", value => { value.cases[3].controllerAuthority.rows[0].company_id = "00000000-0000-4000-8000-000000099999"; }, /authority company mismatch/], + ["h4d-wrong-authority-run", value => { value.cases[3].controllerAuthority.rows[0].run_id = value.cases[5].runIds[0]; }, /authority run set mismatch/], + ]) { + const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0], + value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString()); + change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64"); + const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`); + const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, + `--original-h4d-proof=${file}`, `--original-h4d-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]); + assert.notEqual(attempt.status, 0, name); + const rejected = JSON.parse(await readFile(output, "utf8")); + assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false); + assert.match(rejected.originalH4dConfirmation.reason, message); + } + const badOutput = path.join(directory, "still-misses.json"); + const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`, + `--noise-report=${noise}`, `--output=${badOutput}`, "--require-acceptance"]); + assert.notEqual(bad.status, 0); + const rejected = JSON.parse(await readFile(badOutput, "utf8")); + assert.equal(rejected.acceptancePassed, false); + assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false); + for (const [name, mutate, message] of [ + ["old-h5a-protocol", report => { report.protocolVersion = "v5-complete-observation-original-sla"; }, /strictly equal/], + ["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/], + ["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/], + ["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/], + ["missing-delta", report => { report.cases[3].durable[99].individualDeltas--; }, /individualDeltas|result.events|512/], + ["wrong-workload", report => { report.workloadSha256 = "old"; }, /registration changed/], + ["pw-retry", (_report, owner) => { owner.results[0].retry = 1; }, /Retries/], + ["pw-error", (_report, owner) => { owner.results[0].errors.push({ message: "unexpected API failure" }); }, /deep-equal/], + ["wrong-observation-timeout", (_report, owner) => { owner.timeout = 900000; }, /1200000/], + ]) { + const result = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutate })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(result.status, 0, name); assert.match(result.stderr, message, name); + } + const outer = run(["--candidate-policy=final-b2", `--pairs=${await manifest("outer900", 1000, { duration: 950000 })}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(outer.status, 0, "Observed total duration900 miss cannot qualify final B2"); + const incomplete = JSON.parse(await readFile(await manifest("missing-owner", 1000), "utf8")); + delete incomplete[0].candidatePlaywrightFile; + const missing = run(["--candidate-policy=final-b2", `--pairs=${await save("missing-owner-manifest.json", incomplete)}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(missing.status, 0); assert.match(missing.stderr, /independent owning Playwright/); + const global = JSON.parse(await readFile(await manifest("global-error", 1000), "utf8")); + const globalOwner = JSON.parse(await readFile(global[0].candidatePlaywrightFile, "utf8")); + globalOwner.errors = [{ message: "global teardown failed" }]; + await writeFile(global[0].candidatePlaywrightFile, JSON.stringify(globalOwner)); + const globalFailed = run(["--candidate-policy=final-b2", `--pairs=${await save("global-error-manifest.json", global)}`, + `--noise-report=${noise}`, "--require-acceptance"]); + assert.notEqual(globalFailed.status, 0); assert.match(globalFailed.stderr, /global error/); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + test("browser CLI rejects the three-pairs-above-noise counterexample and ineligible evidence", async () => { const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-gates-")); const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url); @@ -201,9 +428,18 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi ["profile", report => { report.diagnosticOnly = true; }, /Diagnostic profiles/], ["profile-ineligible", report => { report.performanceAcceptanceEligible = false; }, /Profile-only evidence/], ["missing-retained-prelude", report => { report.cases.find(row => row.prelude).preludeProof.individualDeltas--; }, /stored prelude deltas/], - ["legacy-protocol", report => { delete report.protocolVersion; }, /v4 mixed clock protocol/], - ["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /v4 mixed clock protocol/], - ["old-remote-protocol", report => { report.protocolVersion = "v3-supported-remote-original-clock-stream"; }, /v4 mixed clock protocol/], + ["legacy-protocol", report => { delete report.protocolVersion; }, /independent-agent/], + ["old-local-protocol", report => { report.protocolVersion = "v2-compact-markers-original-clock-stream"; report.transport = "local"; }, /independent-agent/], + ["old-remote-protocol", report => { report.protocolVersion = "v3-supported-remote-original-clock-stream"; }, /independent-agent/], + ["old-h4c-geometry", report => { report.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream"; }, /independent-agent/], + ["shared-participant-agent", report => { report.cases[3].controllerAuthority.rows[1].agent_id = report.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/], + ["participant-outside-pool", report => { report.cases[3].controllerAuthority.rows[1].agent_id = "00000000-0000-4000-8000-000000099999"; }, /outside the normal agent pool/], + ["incomplete-agent-pool", report => { report.remote.agentIds.pop(); }, /Incomplete independent/], + ["unfinished-agent-setup", report => { report.remote.agentSetup.complete = false; }, /settle successfully/], + ["rejected-agent-setup", report => { report.remote.agentSetup.attempts[0].status = "rejected"; }, /Rejected agent setup/], + ["agent-setup-concurrency", report => { report.remote.agentSetup.concurrency = 9; }, /strictly equal/], + ["agent-setup-overlap", report => { for (const row of report.remote.agentSetup.attempts.slice(0, 9)) { row.startedAtMs = 3; row.settledAtMs = 39; } }, /exceeded the fixed concurrency/], + ["late-agent-setup", report => { report.remote.agentSetup.settledAtMs = 60; }, /before every case activation/], ["wrong-transport", report => { report.transport = "local"; }, /original local Stop transport/], ["missing-transport-policy", report => { delete report.transportPolicy; }, /explicit transport/], ["fake-remote-stop", report => { report.cases[5].transport = "sandbox-remote"; }, /Wrong per-case transport/], diff --git a/scripts/native-ingestion/h5b/README.md b/scripts/native-ingestion/h5b/README.md new file mode 100644 index 00000000000..bc6aa0690ed --- /dev/null +++ b/scripts/native-ingestion/h5b/README.md @@ -0,0 +1,115 @@ +# Frozen H5b application helpers + +These are exact source snapshots of the four reviewed task helpers. `sources.json` +records their original paths, SHA256 values and the frozen inputs. No helper code +or embedded path was changed for packaging. The owning run receipt identifies the bytes +that actually executed. Copying tools into a delivery commit does not rerun or +change the provenance of earlier evidence. + +The recorded SMALL post-read check used extractor SHA256 +`00930574798e3d56257a4913d82400ccd739b4bb52e9a24d5cb18a36892be970` +(5445 bytes), retained with its raw 23-file artifact manifest. The packaged +extractor is the later reviewed SHA256 +`a052375a80b3abad4513cdf019325e83fed63c67d935f1dc2accba753ce589f0` +(5704 bytes). It adds three existing-history prelude assertions: foreground run +identity, 1000 unique deltas, and verification before epoch with retained head. +It did not produce the SMALL proof. Measured driver bytes are unchanged. + +The executed driver is tag `native-ingestion-h5b`, annotated tag object +`34498d7e09b61493e0f6703c354291c485836298`, commit +`179fe6fa6da1b24ca08c1162301538d3e3788af6`. Its lock SHA256 is +`54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8`. +The delivery lock is different. Do not replace it or describe a delivery checkout +as that executed driver. The [frozen registration](../../../tests/e2e/fixtures/native-ingestion-h5.md) +sets the independent-agent geometry, deadlines and evidence gates. + +## Recorded Linux setup + +Use a disposable, quiet Linux host with the recorded Node/pnpm, PostgreSQL, +Chromium and native Runner inputs. Install and normally build each exact APP +revision independently; retain its build receipt and all four inventory roots. +Keep source, node_modules, TypeScript output and compiled APP output separate. +Only the native Rust target could be reused: all 15 native inputs matched exactly, +including Rust tree `bbe29bba574ce8d5cfddf2f7461ae0eb5983df07`. +The build receipts do not imply three fresh Rust compilations. The three APP +revisions and directory names are in `sources.json`. Raw B2 `ca7cba1` is the unoptimized +batching base. A future Rust candidate needs a new immutable revision, normal +build and qualified inventory; none is supplied here. + +Keep the recorded layout under `/home/ubuntu/native-ingestion`: the independent +179 driver at `harness-h5b`, APP trees at `aligned-baseline`, `aligned-cursors` +and `aligned-batching`, qualified build receipts and inventories in their recorded +locations, and these four helpers in `helpers` under their original basenames. +Verify their copied bytes against `sources.json`. The qualification/readiness +wrappers intentionally retain those paths. Any path-portable derivative must +record its changes and separate hashes. + +Run this snippet from the current tools checkout. The paths retain the recorded +Linux layout. A new driver/output path must be absent before creation. + +```sh +TOOLS_CHECKOUT="$(git rev-parse --show-toplevel)" +DRIVER_ROOT=/home/ubuntu/native-ingestion/harness-h5b +HELPERS_ROOT=/home/ubuntu/native-ingestion/helpers +git -C "$TOOLS_CHECKOUT" show-ref --verify --quiet refs/tags/native-ingestion-h5b || + git -C "$TOOLS_CHECKOUT" fetch origin tag native-ingestion-h5b +test "$(git -C "$TOOLS_CHECKOUT" rev-parse 'native-ingestion-h5b^{commit}')" = 179fe6fa6da1b24ca08c1162301538d3e3788af6 +git -C "$TOOLS_CHECKOUT" worktree add --detach "$DRIVER_ROOT" native-ingestion-h5b +pnpm --dir "$DRIVER_ROOT" install --frozen-lockfile --offline --ignore-scripts +mkdir -p "$HELPERS_ROOT" +for helper in native-ingestion-run-h5b-browser.mjs native-ingestion-linux-h5b-qualification.py native-ingestion-h5b-linux-readiness.mjs native-ingestion-extract-h5b-qualification.py sources.json; do + cp "$TOOLS_CHECKOUT/scripts/native-ingestion/h5b/$helper" "$HELPERS_ROOT/$helper" +done +node "$DRIVER_ROOT/scripts/describe-native-ingestion-h5.mjs" +node "$TOOLS_CHECKOUT/scripts/native-ingestion/h5b/native-ingestion-run-h5b-browser.mjs" +``` + +The last two commands are dry runs. They start no app or performance arm. +The recorded driver install is independent of each APP's normal install/build. +Use the original frozen driver for active comparisons and later same-source H4d +confirmation, even when these source bytes also appear in the delivery PR. + +## Qualification and comparisons + +Only start one released stage in an exclusive quiet window. Unset inherited +`DATABASE_URL`, `PAPERCLIP_RUNNER_STATE_DIR` and diagnostic overrides as the wrapper does. +Use fresh output names; preserve failures and existing output. After the build +and live-inventory checks, the recorded qualification commands are: + +```sh +QUALIFICATION_OUTPUT=/home/ubuntu/native-ingestion/evidence/h5b/qualification/NEW-FULL +python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-linux-h5b-qualification.py --tree=B0 --name=NEW-SMALL +python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-linux-h5b-qualification.py --tree=B0 --full --name=NEW-FULL +python3 /home/ubuntu/native-ingestion/helpers/native-ingestion-extract-h5b-qualification.py "$QUALIFICATION_OUTPUT" --revision=bac53ec3bfeaf6546e7aaeaee516487b30534654 --full +``` + +The extractor's `--full` must match the qualification. It verifies retained +reports and emits qualification-only output. The readiness helper additionally +checks actual B0/B1 inventories, native target, PATH and physical quiet; its +output path must be fresh. It does not start an APP. + +The browser orchestrator prints its five stages without arguments. A stage needs +an explicit byte-frozen `--release=...`, `--stage=...`, `--output-root=...` and +`--execute`. Static `--validate-release` checks do not execute an arm. Release +inputs bind the exact driver, tools, APP/build inventories, stage order and prior +noise reports. Qualify full B0 and forecast duration before fresh B0 and B1 A/A; +freeze both noise results before implementing another product optimization. + +H5b uses 100 normal independent agents, one concurrent run each, and eight setup +workers. Counts, rates and original polls stay fixed. H5 observation has explicit +600s streaming and 1200s invocation limits. The original 120/180/900s outcomes stay +separate; final B2 must meet every original SLA in every arm and later pass H4d +on the same frozen source/geometry/SUT. The 30-minute resource watchdog grants no +SLA extension or performance acceptance. + +Status at the recorded 2026-11-04 05:27:45 UTC launch: B0 SMALL passed functional +qualification; full B0 had started. The later full B0 H5b attempt **failed**: +83 runs succeeded and 17 failed; owning Playwright duration was 742.825s. +Existing-history and Stop were not reached. Authority checks verified 100 distinct +agents, and source clocks verified all 51,200 emissions. Complete canonical +persisted counts are not yet established. Full PostgreSQL causal forensics remain +pending. This is not a passed FULL qualification or an accepted performance arm. + +This source package claims no A/A, browser gain or application acceptance. +The old 2-agent × 50-run failure remains a product limitation. Current run receipts +and the dated benchmark report, not this source package, establish later outcomes. diff --git a/scripts/native-ingestion/h5b/native-ingestion-extract-h5b-qualification.py b/scripts/native-ingestion/h5b/native-ingestion-extract-h5b-qualification.py new file mode 100644 index 00000000000..2db1fc94633 --- /dev/null +++ b/scripts/native-ingestion/h5b/native-ingestion-extract-h5b-qualification.py @@ -0,0 +1,63 @@ +import pathlib,json,base64,argparse,hashlib +p=argparse.ArgumentParser();p.add_argument('directory');p.add_argument('--revision',required=True);p.add_argument('--full',action='store_true');a=p.parse_args();d=pathlib.Path(a.directory);r=json.loads((d/'playwright.json').read_text());attachments=[] +def visit(value): + if isinstance(value,list): + for child in value:visit(child) + elif isinstance(value,dict): + attachments.extend(item for item in value.get('attachments',[])if item.get('name')=='native-ingestion-evidence') + for key,child in value.items(): + if key!='attachments':visit(child) +visit(r);assert len(attachments)==1,len(attachments);item=attachments[0] +raw=base64.b64decode(item['body'])if item.get('body')else pathlib.Path(item['path']).read_bytes();report=json.loads(raw);(d/'native-ingestion-evidence.json').write_bytes(raw) +assert report.get('complete')is True and not report.get('primaryFailure'),report.get('primaryFailure') +assert report['sourceRevision']==report['compiledBuildRevision']==a.revision +assert report['protocolVersion']=='v5b-independent-agents-complete-observation-original-sla' +assert report['observationMode']=='h5b-observe-original-sla-independent-agents' and report['observationComplete'] is True +assert report['measurementSha256']=='452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e' +assert report['transport']=='sandbox-remote-with-local-stop' +assert report['admissionProtocol']=='two-at-a-time-provider-and-clock-readiness-before-common-epoch' +assert report['setupActivationConcurrency']==2 +assert report['harnessProvenance']['sourceRevision']=='179fe6fa6da1b24ca08c1162301538d3e3788af6' +assert report['qualificationOnly']is(not a.full) +assert report['serverArtifact']=='compiled' and report.get('diagnosticOnly')is False +assert report['cleanup']=={'errors':[],'complete':True} +expected=['paced','burst16','burst128','burst512','existing-history','operator-stop']if a.full else['qualification','operator-stop'];assert [c['name']for c in report['cases']]==expected +ids=report['remote']['agentIds'];setup=report['remote']['agentSetup'] +assert len(ids)==(100 if a.full else 3) and len(set(ids))==len(ids) +assert setup['complete'] is True and setup['concurrency']==8 and setup['count']==len(ids) +assert len(setup['attempts'])==len(ids) and all(x['status']=='fulfilled'for x in setup['attempts']) +assert all(ids[x['index']]==x['agentId']for x in setup['attempts']) +rows=[] +for case in report['cases']: + stop=case['name']=='operator-stop';assert case['persistedAfterReload']is True + assert case['setupActivationConcurrency']==2 and case['participants']==case['runs'] + assert sorted(x['index']for x in case['activations'])==list(range(case['runs'])) + assert all(x['status']=='fulfilled'for x in case['activations']) + waves=case['activationWaves'];assert len(waves)==(case['runs']//2) + assert all(1<=len(w['indices'])<=2 and w['statuses']==['fulfilled']*len(w['indices'])for w in waves) + if not a.full and not stop:assert(case['runs'],case['events'],case['intervalMs'])==(3,16,25) + assert len(case['durable'])==case['runs']==len(case['runIds']) + if case['name']=='existing-history': + pre=case['preludeProof'];assert pre['runId']==case['runIds'][0] and pre['individualDeltas']==1000 and pre['uniqueSourceIdentities']==1000 + assert pre['verifiedAtMs']=1000 + assert case['matchedVisibleClocks']==len(case['visible']) and len(case['visible'])>=(3 if stop else case['events']) + authority=case['controllerAuthority'];assert authority['kind']==('host-file-journal'if stop else'database-controller-checkpoint') + assert authority['observedReadOnly']=='on'and authority['observedIsolation']=='repeatable read' + assert len(authority['rows'])==case['runs']and authority['connectionClosedAtMs']int(db['process_identity_seq']) + else: + assert len(set(x['agent_id']for x in case['controllerAuthority']['rows']))==case['runs'] + assert all(x['agent_id']in ids for x in case['controllerAuthority']['rows']) + assert len(case['remoteLeases'])==case['runs'] + assert all(lease['releasedAt']and lease['cleanupStatus']=='success'for lease in case['remoteLeases']) + rows.append({'name':case['name'],'runs':case['runs'],'eventsPerRun':case['events'],'visibleMarkers':len(case['visible']),'individualDeltas':sum(v['individualDeltas']for v in case['durable']),'sourceClocksMatched':sum(v['matchedSourceClocks']for v in case['durable']),'runIds':case['runIds'],'transport':case['transport'],'setupWaves':len(case['activationWaves']),'setupMs':case['allReadyObservedAtMs']-case['setupReadinessStartedAtMs'],'drainThroughTerminalMs':case['drainThroughTerminalMs']}) +summary={'complete':True,'qualificationOnly':True,'performanceAcceptanceEligible':False,'sourceRevision':a.revision,'harnessRevision':report['harnessProvenance']['sourceRevision'],'reportSha256':hashlib.sha256(raw).hexdigest(),'fullFixture':a.full,'cases':rows,'cleanup':report['cleanup']};(d/'independent-qualification-check.json').write_text(json.dumps(summary,indent=2)+'\n');print(json.dumps(summary)) diff --git a/scripts/native-ingestion/h5b/native-ingestion-h5b-linux-readiness.mjs b/scripts/native-ingestion/h5b/native-ingestion-h5b-linux-readiness.mjs new file mode 100644 index 00000000000..39c6afa0cd1 --- /dev/null +++ b/scripts/native-ingestion/h5b/native-ingestion-h5b-linux-readiness.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict'; +import {readFile,writeFile,realpath} from 'node:fs/promises'; +import {execFileSync} from 'node:child_process'; +import {createHash} from 'node:crypto'; +import path from 'node:path'; +const base='/home/ubuntu/native-ingestion',helper=base+'/helpers/native-ingestion-run-h5b-browser.mjs',out=base+'/evidence/h5b/setup'; +const sha=b=>createHash('sha256').update(b).digest('hex'); +assert.equal(sha(await readFile(helper)),'522d4fc718665ef484dec40ff56e463efe11b4ddf0c632c2d3f4811092d63fa6'); +const h=await import(helper); +const quiet=await h.physicalQuietGuard({receiptFile:out+'/linux-quiet-readiness.json',phase:'pre-arm',armStage:'H5b-SETUP-ONLY'}); +const receiptFile=base+'/aligned-4a83-builds-qualified.json',summaryFile=base+'/evidence/aligned-4a83/environment/artifact-inventory-summary.json'; +const receipt=JSON.parse(await readFile(receiptFile,'utf8')),summary=JSON.parse(await readFile(summaryFile,'utf8')); +const results=[]; +for(const [variant,directory] of [['B0','aligned-baseline'],['B1','aligned-cursors']]) { + const row=receipt.results.find(v=>v.variant===variant),i=summary.inventories[variant],native=receipt.nativeStageHashes.find(v=>v.variant===variant); + const descriptor={appRoot:base+'/'+directory,revision:row.revision,qualifiedReceiptFile:receiptFile,qualifiedReceiptSha256:sha(await readFile(receiptFile)),inventorySummaryFile:summaryFile,inventorySummarySha256:sha(await readFile(summaryFile)),manifestFile:path.join(path.dirname(summaryFile),i.manifest),manifestFileSha256:i.manifestSha256,aggregateSha256:i.aggregateSha256,binarySha256:native.binarySha256,rustSourceTree:receipt.rustSourceTree}; + const q=await h.validateVariantBuild(variant,descriptor),live=await h.validateActualInventory(descriptor,q.manifest),ui=await h.assertNoPreferredUiShadow(descriptor.appRoot); + assert.equal(execFileSync('git',['rev-parse','HEAD'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),descriptor.revision); + assert.equal(execFileSync('git',['status','--short'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),''); + assert.equal(JSON.parse(await readFile(path.join(descriptor.appRoot,'server/dist/build-info.json'),'utf8')).commit,descriptor.revision); + const target=path.join(descriptor.appRoot,'packages/paperclip-runner/runner/target'),binary=path.join(target,'release/paperclip-runnerd'); + assert.equal(await realpath(target),native.targetRealPath);assert.equal(sha(await readFile(binary)),native.binarySha256); + results.push({variant,descriptor,live,ui,nativeTargetRealPath:await realpath(target)}); +} +const pathNode=execFileSync('node',['-p','process.execPath'],{encoding:'utf8'}).trim(); +assert.equal(await realpath(pathNode),await realpath(process.execPath)); +const data={setupOnly:true,noSutBrowserProviderOrEvents:true,performanceAcceptanceEligible:false,atMs:Date.now(),helperSha256:sha(await readFile(helper)),harnessRevision:'179fe6fa6da1b24ca08c1162301538d3e3788af6',quiet,path:{PATH:process.env.PATH,nodeExecutable:process.execPath,pathResolvedNode:pathNode,nodeVersion:process.version},results}; +await writeFile(out+'/linux-readiness.json',JSON.stringify(data,null,2)+'\n',{flag:'wx'}); +console.log(JSON.stringify({complete:true,setupOnly:true,noSutBrowserProviderOrEvents:true,variants:results.map(v=>({variant:v.variant,fileCount:v.live.fileCount,aggregate:v.live.aggregateSha256,uiShadowAbsent:v.ui.preferredIndexAbsent})),quiet:quiet.passed,nodeVersion:process.version})); diff --git a/scripts/native-ingestion/h5b/native-ingestion-linux-h5b-qualification.py b/scripts/native-ingestion/h5b/native-ingestion-linux-h5b-qualification.py new file mode 100644 index 00000000000..1517b5a70d7 --- /dev/null +++ b/scripts/native-ingestion/h5b/native-ingestion-linux-h5b-qualification.py @@ -0,0 +1,58 @@ +import pathlib,subprocess,json,os,socket,datetime,hashlib,traceback,argparse +p=argparse.ArgumentParser();p.add_argument('--full',action='store_true');p.add_argument('--tree',choices=['B0','B1','B2'],default='B2');p.add_argument('--name',required=True);a=p.parse_args() +base=pathlib.Path('/home/ubuntu/native-ingestion');repo=base/'paperclip';h=base/'harness-h5b';out=base/'evidence/h5b/qualification'/a.name;out.mkdir(parents=True,exist_ok=False) +pins={'B0':('aligned-baseline','bac53ec3bfeaf6546e7aaeaee516487b30534654'),'B1':('aligned-cursors','07061a601cdc20603c8c7a46f9ddbad7ac5f5939'),'B2':('aligned-batching','ca7cba171cd5198ed569d75012edc858041cbc44')};app=base/pins[a.tree][0] +ref='179fe6fa6da1b24ca08c1162301538d3e3788af6';protocol='452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e';comparator='0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168';lock='54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8' +r={'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'qualificationOnly':True,'performanceAcceptanceEligible':False,'fullFixture':a.full,'sourceRevision':pins[a.tree][1],'harnessRevision':ref,'steps':[]} +def now():return datetime.datetime.now(datetime.timezone.utc).isoformat() +def save():(out/'owner-receipt.json').write_text(json.dumps(r,indent=2)+'\n') +def cmd(name,args,cwd,env=None,timeout=None): + row={'name':name,'argv':list(map(str,args)),'cwd':str(cwd),'startedAt':now()};r['steps'].append(row);save() + with(out/(name+'.log')).open('xb')as log:result=subprocess.run(args,cwd=cwd,env=env,stdout=log,stderr=subprocess.STDOUT,timeout=timeout) + row.update({'exit':result.returncode,'finishedAt':now()});save();assert result.returncode==0,(name,result.returncode) +try: + assert(base/'aligned-4a83-builds.exit').read_text().strip()=='0' + qualified=json.loads((base/'aligned-4a83-builds-qualified.json').read_text()) + assert {row['variant']:row['revision'] for row in qualified['results']}=={v:pin for v,(directory,pin) in pins.items()} + rows=[] + for q in pathlib.Path('/proc').iterdir(): + if not q.name.isdigit():continue + try: + exe=os.readlink(q/'exe') + if pathlib.Path(exe).name in['node','postgres','paperclip-runner','rustc','cargo','chrome','chromium','chrome-headless-shell']:rows.append({'pid':int(q.name),'executable':exe}) + except OSError:pass + assert not rows,rows + for directory,pin in pins.values(): + root=base/directory;assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=root,text=True).strip()==pin + assert not subprocess.check_output(['git','status','--short'],cwd=root,text=True).strip() + assert json.loads((root/'server/dist/build-info.json').read_text())['commit']==pin + assert h.exists(), 'Stage the exact independent driver before qualification' + assert not(h/'node_modules').is_symlink();assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=h,text=True).strip()==ref + assert not subprocess.check_output(['git','status','--short'],cwd=h,text=True).strip() + files=['native-event-ingestion.journey.ts','playwright-native-ingestion.config.ts','fixtures/in-feed-codex.mjs','fixtures/native-ingestion-bin/codex','fixtures/composer-stop-codex-loader.mjs','fixtures/native-ingestion-bootstrap.mjs','fixtures/native-ingestion-remote-server.mjs','fixtures/native-ingestion-protocol-qualification.mjs','fixtures/native-ingestion-h5.mjs'] + assert hashlib.sha256(b''.join((h/'tests/e2e'/f).read_bytes()for f in files)).hexdigest()==protocol + assert hashlib.sha256((h/'scripts/compare-native-ingestion-browser.ts').read_bytes()).hexdigest()==comparator + assert hashlib.sha256((h/'pnpm-lock.yaml').read_bytes()).hexdigest()==lock + assert hashlib.sha256((h/'tests/e2e/fixtures/native-ingestion-h5.md').read_bytes()).hexdigest()=='60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54' + r['resourceWatchdogMillis']=1800000;r['resourceWatchdogScope']='Resource-only including startup/test/teardown; original SLA and H5 observation unchanged';r['protocolSha256']=protocol;r['comparatorSha256']=comparator;r['driverLockSha256']=lock + with socket.socket()as s:s.bind(('127.0.0.1',0));port=s.getsockname()[1] + env=dict(os.environ) + for key in ['DATABASE_URL','PAPERCLIP_RUNNER_STATE_DIR','PAPERCLIP_INGESTION_SERVER_COMMAND','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR','PAPERCLIP_INGESTION_BROWSER_CAPTURE','PAPERCLIP_INGESTION_SMOKE','NODE_OPTIONS','PAPERCLIP_INGESTION_OBSERVATION_MODE']:env.pop(key,None) + binary=str(app/'packages/paperclip-runner/runner/target/release/paperclip-runnerd') + overrides={'PAPERCLIP_INGESTION_OBSERVATION_MODE':'h5b-observe-original-sla-independent-agents','PAPERCLIP_INGESTION_TRANSPORT':'sandbox-remote-with-local-stop','PAPERCLIP_INGESTION_APP_ROOT':str(app),'PAPERCLIP_INGESTION_EVIDENCE_DIR':str(out),'PAPERCLIP_E2E_PORT':str(port),'PAPERCLIP_RUNNER_BINARY':binary,'PAPERCLIP_RUNNER_REMOTE_BINARY_PATH':binary,'PAPERCLIP_INGESTION_VARIANT':a.name+'-FUNCTIONAL-QUALIFICATION-ONLY','PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES':'Independent H5b frozen offline ignore-scripts driver install; application owns independent normal locked install/build'} + if not a.full:overrides['PAPERCLIP_INGESTION_SMOKE']='1' + env.update(overrides);r['environmentOverrides']=overrides;save();subprocess.run(['sync'],check=True) + cmd('playwright',['timeout','--signal=INT','--kill-after=40s','30m','node','node_modules/@playwright/test/cli.js','test','--config','tests/e2e/playwright-native-ingestion.config.ts'],h,env,1845) + r['playwrightPassed']=True;r['finishedAt']=now();save() +except BaseException as error:r.update({'passed':False,'error':str(error),'traceback':traceback.format_exc(),'finishedAt':now()});save();(out/'owner.exit').write_text('1\n');raise + +finally: + helper=base/'helpers/native-ingestion-run-h5b-browser.mjs' + guard="import net from 'node:net'; import {physicalQuietGuard} from '"+str(helper)+"'; const s=net.createServer(); await new Promise((ok,bad)=>{s.once('error',bad);s.listen(Number(process.argv[2]),'127.0.0.1',ok)}); await new Promise(ok=>s.close(ok)); await physicalQuietGuard({receiptFile:process.argv[1],phase:'post-arm',armStage:'H5b-QUALIFICATION-ONLY'});" + try: + result=subprocess.run(['node','--input-type=module','-e',guard,str(out/'physical-quiet-after.json'),str(r['environmentOverrides']['PAPERCLIP_E2E_PORT'])],check=True,capture_output=True,text=True) + (out/'port-after.json').write_text(json.dumps({'complete':True,'port':r['environmentOverrides']['PAPERCLIP_E2E_PORT'],'checkedAt':now(),'nodeBindAndListenPassed':True})+'\n') + if r.get('playwrightPassed'): + r['passed']=True;r['cleanupComplete']=True;save();(out/'owner.exit').write_text('0\n') + except BaseException as cleanup_error: + r['externalQuietFailure']=str(cleanup_error);r['passed']=False;r['cleanupComplete']=False;save();(out/'owner.exit').write_text('1\n');raise diff --git a/scripts/native-ingestion/h5b/native-ingestion-run-h5b-browser.mjs b/scripts/native-ingestion/h5b/native-ingestion-run-h5b-browser.mjs new file mode 100644 index 00000000000..13668c98195 --- /dev/null +++ b/scripts/native-ingestion/h5b/native-ingestion-run-h5b-browser.mjs @@ -0,0 +1,652 @@ +import assert from 'node:assert/strict'; +import { spawn, execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { readFile, writeFile, mkdir, open, access, readdir, realpath, stat, lstat } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import net from 'node:net'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { gunzipSync } from 'node:zlib'; + +// Prospective H5 application orchestrator. No launch without --execute and a +// byte-frozen root release. Original ce33 helper SHA is provenance only: +// 2244e238642caafd488a7c86fb463376c8459ee0f06f8054555407a174698fa1. +export const MODE = 'h5b-observe-original-sla-independent-agents'; +export const PROTOCOL = 'v5b-independent-agents-complete-observation-original-sla'; +export const WORKLOAD_SHA256 = '9d34b1511610cd5fcc73f8d734114e624105a3ee3b1e45c2b4d1cb721cad048c'; +const H5_REVISION = '179fe6fa6da1b24ca08c1162301538d3e3788af6'; +const OLD_NATIVE_TARGET = '/home/ubuntu/native-ingestion/aligned-baseline/packages/paperclip-runner/runner/target'; +const H5_PROTOCOL_SHA256 = '452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e'; +const H5_COMPARATOR_SHA256 = '0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168'; +const H5_INVENTORY_HELPER_SHA256 = '76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9'; +const H5_PROTOCOL_DOCUMENT_SHA256 = '60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54'; +const INVENTORY_ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src']; +export const RESOURCE_WATCHDOG_MILLIS = 30 * 60_000; +export const STAGES = Object.freeze(['browser-aa-b0', 'browser-aa-b1', 'browser-b0-b1', 'browser-b1-b2', 'browser-b0-b2']); +export const FINALIZE_STAGE = 'browser-finalize-h4d'; +const protocolFiles = ['native-event-ingestion.journey.ts', 'playwright-native-ingestion.config.ts', 'fixtures/in-feed-codex.mjs', + 'fixtures/native-ingestion-bin/codex', 'fixtures/composer-stop-codex-loader.mjs', 'fixtures/native-ingestion-bootstrap.mjs', + 'fixtures/native-ingestion-remote-server.mjs', 'fixtures/native-ingestion-protocol-qualification.mjs', 'fixtures/native-ingestion-h5.mjs']; +const BASELINES = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654', B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' }); +const OLD_B2 = 'ca7cba171cd5198ed569d75012edc858041cbc44'; +const OLD_RECEIPT = 'a1259ba760f3cbbe44817e333f9f620bcf3769fb0a6c16f9ce2f0f5b2ed60d94'; +const OLD_SUMMARY = 'd3766fc4206d569d10fd02224433f1e9a2caa6992a7c62216e80af368aa4e44d'; +const OLD_BINARY = 'abb65bdc073991e428bd37f9809603ee7564543cca0ed0305dc231137c93070f'; +const OLD_RUST = 'bbe29bba574ce8d5cfddf2f7461ae0eb5983df07'; +const OLD_INVENTORIES = Object.freeze({ + B0: { manifestFileSha256: 'a1c797a75dd3174f414fc5529acf56783e1a3dfe7b22f1e460a87b60f6bff01d', aggregateSha256: 'a50720be2ce53f9222187d63609c5936f94b6a00c60df2ca112261abd1bf921e' }, + B1: { manifestFileSha256: 'df48e5c511a2f31e31180e05f18e2322ca4f384ea06f6f96a57de24d39849ef8', aggregateSha256: 'bca5b6fad62e70631c135a9ff377efeb587aba17ef0842c8fd82804ed99a81ab' }, +}); +const LOCK_SHA256 = '54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8'; +const GATES_SHA256 = 'ded8bcbb8f05732348f6520b9c5e41b7c35bc098997eb263a9dcb174eede75fc'; +const selfFile = fileURLToPath(import.meta.url); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const getJson = async file => JSON.parse(await readFile(file, 'utf8')); +const saveJson = async (file, value, fresh = false) => writeFile(file, JSON.stringify(value, null, 2) + '\n', fresh ? { flag: 'wx' } : undefined); +const digest = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{64}$/.test(value), `Missing exact ${name} SHA-256`); +const revision = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{40}$/.test(value), `Missing immutable full ${name} revision`); +const absolute = (value, name) => assert.ok(typeof value === 'string' && path.isAbsolute(value), `Missing absolute ${name} path`); +const option = name => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3); + +export function stagePlan(stage) { + assert.ok(STAGES.includes(stage) || stage === FINALIZE_STAGE, 'Select one registered H5 stage'); + if (stage === FINALIZE_STAGE) return { stage, requiredVariants: ['B0', 'B1', 'B2'], priorStages: [...STAGES], fullArms: 0, finalizationOnly: true }; + const [baseline, candidate] = stage === STAGES[0] ? ['B0', 'B0'] : stage === STAGES[1] ? ['B1', 'B1'] + : stage === STAGES[2] ? ['B0', 'B1'] : stage === STAGES[3] ? ['B1', 'B2'] : ['B0', 'B2']; + const aa = baseline === candidate; + return { stage, baseline, candidate, aa, expectation: stage === STAGES[2] ? 'nonregression' : 'improvement', + candidatePolicy: aa ? null : stage === STAGES[2] ? 'cursor-nonregression' : 'final-b2', + requiredVariants: [...new Set([baseline, candidate])], priorStages: STAGES.slice(0, STAGES.indexOf(stage)), + pairs: 5, fullArms: 10, noiseStage: aa ? null : `browser-aa-${baseline.toLowerCase()}` }; +} + +async function hashedJson(file, expected, name, compressed = false) { + absolute(file, name); digest(expected, name); + const bytes = await readFile(file); assert.equal(hash(bytes), expected, `${name} bytes changed`); + return { bytes, value: JSON.parse((compressed ? gunzipSync(bytes, { maxOutputLength: 128 * 1024 * 1024 }) : bytes).toString('utf8')) }; +} + +export async function validateVariantBuild(variant, descriptor) { + assert.ok(descriptor && typeof descriptor === 'object', `Required ${variant} qualified build is missing`); + absolute(descriptor.appRoot, `${variant} appRoot`); revision(descriptor.revision, variant); + digest(descriptor.binarySha256, `${variant} binary`); revision(descriptor.rustSourceTree, `${variant} Rust source tree`); + const receipt = (await hashedJson(descriptor.qualifiedReceiptFile, descriptor.qualifiedReceiptSha256, `${variant} qualified receipt`)).value; + const summary = (await hashedJson(descriptor.inventorySummaryFile, descriptor.inventorySummarySha256, `${variant} inventory summary`)).value; + const manifest = (await hashedJson(descriptor.manifestFile, descriptor.manifestFileSha256, `${variant} inventory manifest`, true)).value; + digest(descriptor.aggregateSha256, `${variant} aggregate inventory`); + const rows = receipt.results?.filter(row => row.variant === variant); + assert.equal(rows?.length, 1, `${variant} qualified receipt must select this variant exactly once`); + assert.equal(rows[0].revision, descriptor.revision, `${variant} qualified receipt revision mismatch`); + assert.equal(rows[0].compiledBuildRevision, descriptor.revision, `${variant} qualified compiled revision mismatch`); + assert.equal(rows[0].sourceClean, true, `${variant} qualified source was not clean`); + const native = receipt.nativeStageHashes?.filter(row => row.variant === variant); + assert.equal(native?.length, 1, `${variant} qualified native stage is missing`); + assert.equal(native[0].binarySha256, descriptor.binarySha256, `${variant} native stage binary mismatch`); + assert.equal(receipt.binarySha256, descriptor.binarySha256, `${variant} receipt binary mismatch`); + assert.equal(receipt.rustSourceTree, descriptor.rustSourceTree, `${variant} Rust source identity mismatch`); + assert.equal(receipt.rustInputObjects?.runner, descriptor.rustSourceTree, `${variant} qualified Rust input objects are missing`); + absolute(native[0].targetRealPath, `${variant} qualified native target`); + for (const [file, object] of Object.entries(receipt.rustInputObjects)) { + assert.ok(file && !path.posix.isAbsolute(file) && path.posix.normalize(file) === file && !file.includes('\\') && !file.split('/').includes('..'), 'Qualified Rust input path is invalid'); + revision(object, `${variant} Rust input ${file}`); + } + assert.equal(receipt.artifactInventorySummary, descriptor.qualifiedInventorySummaryReference ?? descriptor.inventorySummaryFile, `${variant} receipt points to another inventory summary`); + assert.equal(manifest.variant, variant); assert.equal(manifest.revision, descriptor.revision); + assert.equal(manifest.compiledBuildRevision, descriptor.revision); assert.equal(manifest.aggregateSha256, descriptor.aggregateSha256); + assert.deepEqual(manifest.roots, INVENTORY_ROOTS); + assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount && manifest.fileCount > 0); + const row = summary.inventories?.[variant]; assert.ok(row, `${variant} inventory summary entry missing`); + assert.equal(row.manifest, path.basename(descriptor.manifestFile)); assert.equal(row.manifestSha256, descriptor.manifestFileSha256); + assert.equal(row.aggregateSha256, descriptor.aggregateSha256); assert.equal(row.fileCount, manifest.fileCount); + if (Object.hasOwn(BASELINES, variant)) { + assert.equal(descriptor.revision, BASELINES[variant], 'Original baseline reference must remain unchanged'); + assert.equal(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'B0/B1 must use the original, unmodified aligned4a83 qualified receipt'); + assert.equal(descriptor.inventorySummarySha256, OLD_SUMMARY); assert.equal(descriptor.binarySha256, OLD_BINARY); + assert.equal(descriptor.rustSourceTree, OLD_RUST); + for (const [key, value] of Object.entries(OLD_INVENTORIES[variant])) assert.equal(descriptor[key], value, `Original ${variant} inventory changed`); + } else { + assert.equal(variant, 'B2'); + assert.ok(![...Object.values(BASELINES), OLD_B2].includes(descriptor.revision), 'Final B2 needs its new actual immutable product revision'); + assert.notEqual(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'Final B2 needs its own new qualified receipt'); + assert.notEqual(descriptor.binarySha256, OLD_BINARY, 'Final Rust B2 needs its newly qualified binary'); + assert.notEqual(descriptor.rustSourceTree, OLD_RUST, 'Final Rust B2 needs its newly qualified Rust inputs'); + assert.notEqual(native[0].targetRealPath, OLD_NATIVE_TARGET, 'Final Rust B2 must not reuse the original B0/B1 target'); + } + return { receipt, summary, manifest }; +} + +// Match the existing inventory generator and H5 helper exactly: Unicode code +// point order, Python ensure_ascii JSON with sorted bytes/path/sha256 keys. +function pythonOrder(left, right) { + const a = [...left], b = [...right]; + for (let index = 0; index < Math.min(a.length, b.length); index++) { + const difference = a[index].codePointAt(0) - b[index].codePointAt(0); + if (difference) return difference; + } + return a.length - b.length; +} +const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 }) + .replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`); +function inventoryPath(value) { + assert.ok(typeof value === 'string' && value.length && !value.includes('\\') && !value.includes('\0') + && !path.posix.isAbsolute(value) && path.posix.normalize(value) === value && !value.split('/').includes('..') + && INVENTORY_ROOTS.some(root => value.startsWith(`${root}/`)), 'Inventory filename escapes registered roots'); + return value; +} +export async function validateActualInventory(build, manifest) { + const startedAtMs = Date.now(), started = performance.now(), canonicalRoot = await realpath(build.appRoot); + assert.deepEqual(manifest.roots, INVENTORY_ROOTS); + const expected = new Map(), canonicalFiles = new Set(); + for (const row of manifest.files) { + const name = inventoryPath(row.path); + assert.equal(expected.has(name), false, 'Duplicate inventory filename'); expected.set(name, row); + digest(row.sha256, 'inventory file'); assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0); + } + assert.equal(expected.size, manifest.fileCount, 'Inventory file count is incomplete'); + const actualNames = [], canonicalDirectories = new Set(); + const walk = async relative => { + const directory = path.join(build.appRoot, relative), canonical = await realpath(directory); + assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory directory resolves outside APP_ROOT'); + assert.equal(canonicalDirectories.has(canonical), false, 'Inventory directory aliases another registered directory'); canonicalDirectories.add(canonical); + for (const entry of await readdir(directory, { withFileTypes: true })) { + const child = `${relative}/${entry.name}`; + if (entry.isDirectory()) await walk(child); + else { + const file = path.join(build.appRoot, child), metadata = await lstat(file); + if (metadata.isSymbolicLink()) assert.equal((await stat(file)).isDirectory(), false, 'Inventory directory symlink is not a frozen file'); + assert.ok(metadata.isFile() || metadata.isSymbolicLink(), 'Inventory includes a nonregular entry'); + inventoryPath(child); actualNames.push(child); + } + } + }; + for (const root of INVENTORY_ROOTS) await walk(root); + actualNames.sort(pythonOrder); + assert.deepEqual(actualNames, [...expected.keys()].sort(pythonOrder), 'Actual APP_ROOT file set differs from frozen inventory'); + const actualRows = new Array(actualNames.length); let next = 0; + await Promise.all(Array.from({ length: Math.min(8, actualNames.length) }, async () => { + for (;;) { + const index = next++; if (index >= actualNames.length) return; + const name = actualNames[index], file = path.join(build.appRoot, name), canonical = await realpath(file); + assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory file resolves outside APP_ROOT'); + assert.equal(canonicalFiles.has(canonical), false, 'Inventory filenames alias the same canonical file'); canonicalFiles.add(canonical); + const before = await stat(file); assert.ok(before.isFile(), 'Inventory evidence must be a regular file'); + const content = createHash('sha256'); for await (const bytes of createReadStream(file)) content.update(bytes); + const after = await stat(file); + assert.equal(after.size, before.size, 'Artifact changed during inventory hashing'); assert.equal(after.mtimeMs, before.mtimeMs, 'Artifact changed during inventory hashing'); + const row = { bytes: after.size, path: name, sha256: content.digest('hex') }, frozen = expected.get(name); + assert.equal(row.bytes, frozen.bytes, `Actual artifact size changed: ${name}`); assert.equal(row.sha256, frozen.sha256, `Actual artifact bytes changed: ${name}`); + actualRows[index] = row; + } + })); + const aggregateSha256 = hash(Buffer.from(`${actualRows.map(pythonRow).join('\n')}\n`, 'utf8')); + assert.equal(aggregateSha256, manifest.aggregateSha256, 'Actual APP_ROOT aggregate differs from frozen inventory'); + assert.equal(aggregateSha256, build.aggregateSha256, 'Actual APP_ROOT aggregate differs from qualified build'); + return { appRoot: build.appRoot, canonicalAppRoot: canonicalRoot, fileCount: actualRows.length, aggregateSha256, + startedAtMs, completedAtMs: Date.now(), elapsedMs: performance.now() - started, + scope: 'Live registered file set, canonical containment/uniqueness, size and SHA checked before this arm; not an assertion of future immutability' }; +} + +function harnessFields(release) { + return Object.fromEntries(['harnessRevision', 'protocolSha256', 'comparatorSha256', 'inventoryHelperSha256', 'protocolDocumentSha256', 'lockSha256', 'gatesSha256', 'workloadSha256', 'observationMode', 'protocolVersion'].map(key => [key, release[key]])); +} + +export async function loadRelease(releaseFile, stage, outputRoot) { + absolute(releaseFile, 'root release'); absolute(outputRoot, 'output root'); + const releaseBytes = await readFile(releaseFile), release = JSON.parse(releaseBytes); + const plan = stagePlan(stage); + assert.equal(release.authorized, true, 'Root release is not authorized'); assert.equal(release.stage, stage); + assert.equal(release.outputRoot, outputRoot, 'Root release freezes a different exact output path'); + assert.equal(release.orchestratorSha256, hash(await readFile(selfFile)), 'Orchestrator changed after release'); + absolute(release.harnessRoot, 'H5 harness'); revision(release.harnessRevision, 'H5 harness'); + assert.equal(release.harnessRevision, H5_REVISION, 'Use the actual independently frozen H5 full revision'); + assert.equal(release.observationMode, MODE); assert.equal(release.protocolVersion, PROTOCOL); + assert.equal(release.workloadSha256, WORKLOAD_SHA256); + assert.equal(release.resourceWatchdogMillis, RESOURCE_WATCHDOG_MILLIS, 'Resource watchdog registration changed; this is not an SLA'); + assert.equal(release.lockSha256, LOCK_SHA256); assert.equal(release.gatesSha256, GATES_SHA256); + for (const [key, expected] of [['protocolSha256', H5_PROTOCOL_SHA256], ['comparatorSha256', H5_COMPARATOR_SHA256], ['inventoryHelperSha256', H5_INVENTORY_HELPER_SHA256], ['protocolDocumentSha256', H5_PROTOCOL_DOCUMENT_SHA256]]) + assert.equal(release[key], expected, `Reviewed H5 source pin changed: ${key}`); + for (const variant of plan.requiredVariants) await validateVariantBuild(variant, release.perVariantBuilds?.[variant]); + const context = { releaseFile, releaseBytesSha256: hash(releaseBytes), release, plan, out: outputRoot, commands: [], commandIndex: 0 }; + await checkPriorStages(context); + if (plan.noiseStage) await checkNoise(context); + if (plan.finalizationOnly) await checkFinalizationInputs(context); + return context; +} + +async function checkReleaseBytes(context) { + assert.equal(hash(await readFile(context.releaseFile)), context.releaseBytesSha256, 'Exact root release bytes changed'); + assert.equal(hash(await readFile(selfFile)), context.release.orchestratorSha256, 'Orchestrator changed after release'); + if (context.rootReleaseCopy) assert.equal(hash(await readFile(context.rootReleaseCopy)), context.releaseBytesSha256, 'Preserved root release bytes changed'); +} + +function checkComparisonIdentity(report, context) { + assert.equal(report.complete, true); assert.equal(report.observationMode, MODE); assert.equal(report.protocolVersion, PROTOCOL); + assert.equal(report.workloadSha256, WORKLOAD_SHA256); assert.equal(report.measurementSha256, context.release.protocolSha256); + assert.equal(report.comparatorSha256, context.release.comparatorSha256); assert.equal(report.gatesSha256, GATES_SHA256); + assert.equal(report.pairCount, 5); +} + +async function checkPriorStages(context) { + for (const stage of context.plan.priorStages) { + const pin = context.release.priorStageReports?.[stage]; assert.ok(pin, `Stage order requires frozen ${stage} first`); + assert.equal(pin.file, path.join(context.out, `${stage}.json`), 'Prior stage report path changed'); + const report = (await hashedJson(pin.file, pin.sha256, stage)).value; + checkComparisonIdentity(report, context); + const priorPlan = stagePlan(stage); + assert.equal(report.phase, priorPlan.aa ? 'aa' : 'compare'); + assert.equal(report.baselineRevision, BASELINES[priorPlan.baseline]); + const candidateRevision = BASELINES[priorPlan.candidate] ?? context.release.perVariantBuilds.B2?.revision; + assert.equal(report.candidateRevision, candidateRevision, 'Prior stage used a different measured candidate revision'); + for (const [arm, variant] of [['baseline', priorPlan.baseline], ['candidate', priorPlan.candidate]]) { + const expected = context.release.perVariantBuilds?.[variant] ?? (BASELINES[variant] ? { ...OLD_INVENTORIES[variant], qualifiedReceiptSha256: OLD_RECEIPT, inventorySummarySha256: OLD_SUMMARY } : undefined); + assert.ok(expected, 'Prior candidate qualified build is unavailable'); + for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256']) + assert.equal(report.artifactInventories?.[arm]?.[key], expected[key], 'Prior stage changed a measured qualified inventory'); + } + if (!priorPlan.aa) assert.equal(report.performancePassed, true, `Prior ${stage} performance failed`); + } +} + +export async function checkNoise(context) { + const { release, plan, out } = context; + const expected = path.join(out, `${plan.noiseStage}.json`); + assert.equal(release.noiseReportFile, expected, 'Use the same exact frozen A/A noise path'); + const noise = (await hashedJson(expected, release.noiseReportSha256, 'Reviewed A/A noise report')).value; + checkComparisonIdentity(noise, context); assert.equal(noise.phase, 'aa'); + assert.equal(noise.baselineRevision, release.perVariantBuilds[plan.baseline].revision); + assert.equal(noise.candidateRevision, noise.baselineRevision); + assert.equal(noise.h5Assessments?.length, 5, 'Reviewed A/A must contain all five complete assessments'); + assert.ok(noise.measurements?.length > 0 && noise.measurements.every(row => Number.isFinite(row.aaNoiseEnvelopeRelative) && row.aaNoiseEnvelopeRelative >= 0), 'A/A noise envelope is invalid'); + const build = release.perVariantBuilds[plan.baseline]; + for (const arm of ['baseline', 'candidate']) for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256']) + assert.equal(noise.artifactInventories?.[arm]?.[key], build[key], 'Frozen A/A baseline inventory differs from required build'); + return noise; +} + +const QUIET_SETTLE_MILLIS = 30_000, QUIET_POLL_MILLIS = 1_000, QUIET_PS_TIMEOUT_MILLIS = 2_000; +const QUIET_SCHEMA = 'paperclip.native-ingestion.physical-quiet.v1'; +const QUIET_SCOPE = 'Dedicated Linux process-name guard for specified Node/PostgreSQL/Runner/browser/Rust-build names only; not a claim that every possible host activity is absent. Qualified H5 Stop proof separately covers provider/sleep control groups.'; +const QUIET_ZOMBIE_POLICY = 'Strict: matched zombie/dead rows remain blockers until absent; no automatic killing or silent zombie exclusion.'; +function processFamily(comm) { + if (/^(node|node-MainThread)$/.test(comm)) return 'node'; + if (/^(postgres|postmaster|pg_ctl|initdb)$/.test(comm)) return 'postgresql'; + if (/^(paperclip-runne|paperclip-runner.*|Runner)$/.test(comm)) return 'runner'; + if (/^(chrome|headless|chromium)/i.test(comm)) return 'browser'; + if (/^(rustc|cargo)$/.test(comm)) return 'rust-build'; + return null; +} +export function classifyLinuxProcesses(text, orchestratorPid = process.pid) { + assert.ok(typeof text === 'string' && text.trim(), 'Linux ps returned no process rows'); + assert.ok(Number.isSafeInteger(orchestratorPid) && orchestratorPid > 0, 'Invalid orchestrator PID'); + const seen = new Set(), matched = []; let rowCount = 0; + for (const line of text.trim().split(/\r?\n/)) { + const fields = line.match(/^\s*(\d+)\s+(\d+)\s+(.+?)\s+([DIRSTtWXZPK])\s*$/); + assert.ok(fields, 'Malformed Linux ps PID/PPID/comm/state row'); + const row = { pid: Number(fields[1]), ppid: Number(fields[2]), comm: fields[3].trim(), state: fields[4] }; + assert.ok(Number.isSafeInteger(row.pid) && row.pid > 0 && Number.isSafeInteger(row.ppid) && row.ppid >= 0 && row.comm.length > 0 && row.comm.length <= 256 && !row.comm.includes('\0'), 'Malformed Linux ps process fields'); + assert.equal(seen.has(row.pid), false, 'Duplicate Linux ps PID'); seen.add(row.pid); rowCount++; + const family = processFamily(row.comm); + if (family) matched.push({ ...row, family, allowed: row.pid === orchestratorPid && !['Z', 'X'].includes(row.state) }); + } + assert.ok(seen.has(orchestratorPid), 'Linux ps omitted the current orchestrator PID'); + return { rowCount, matched, blockers: matched.filter(row => !row.allowed) }; +} +function linuxProcessRows(timeoutMillis) { + return execFileSync('/bin/ps', ['-eo', 'pid=,ppid=,comm=,state='], { + encoding: 'utf8', timeout: timeoutMillis, maxBuffer: 2 * 1024 * 1024, env: { LC_ALL: 'C' }, + }); +} +export async function physicalQuietGuard({ receiptFile, phase, armStage, primaryFailureAlreadyPresent = false, + platform = process.platform, orchestratorPid = process.pid, readPs = linuxProcessRows, + now = () => performance.now(), pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)) }) { + absolute(receiptFile, 'physical quiet receipt'); + const started = now(), receipt = { schema: QUIET_SCHEMA, scope: QUIET_SCOPE, phase, armStage, platform, orchestratorPid, + fields: ['pid', 'ppid', 'comm', 'state'], zombiePolicy: QUIET_ZOMBIE_POLICY, + resourceSettleLimitMillis: QUIET_SETTLE_MILLIS, pollIntervalMillis: QUIET_POLL_MILLIS, psTimeoutMillis: QUIET_PS_TIMEOUT_MILLIS, + primaryFailureAlreadyPresent, startedAtMs: Date.now(), passed: false, observations: [] }; + let failure; + try { + assert.equal(platform, 'linux', 'Accepted H5 physical quiet guard requires Linux'); + assert.ok(['pre-arm', 'post-arm'].includes(phase), 'Unknown physical quiet phase'); + for (;;) { + const remaining = QUIET_SETTLE_MILLIS - (now() - started); + assert.ok(remaining > 0, 'Physical quiet settle deadline expired; stop for cleanup'); + const observed = classifyLinuxProcesses(readPs(Math.max(1, Math.min(QUIET_PS_TIMEOUT_MILLIS, Math.floor(remaining)))), orchestratorPid); + receipt.observations.push({ atMs: Date.now(), elapsedMillis: now() - started, ...observed }); + assert.ok(now() - started < QUIET_SETTLE_MILLIS, 'Physical quiet settle deadline expired; stop for cleanup'); + if (observed.blockers.length === 0) { receipt.passed = true; break; } + const wait = Math.min(QUIET_POLL_MILLIS, QUIET_SETTLE_MILLIS - (now() - started)); + assert.ok(wait > 0, 'Physical quiet settle deadline expired; stop for cleanup'); + await pause(wait); + } + } catch (error) { + failure = error; + receipt.failure = { name: error instanceof Error ? error.name : 'UnknownError', message: error instanceof Error ? error.message : String(error) }; + } + receipt.completedAtMs = Date.now(); receipt.elapsedMillis = now() - started; + const bytes = Buffer.from(JSON.stringify(receipt, null, 2) + '\n'); + await writeFile(receiptFile, bytes, { flag: 'wx' }); + const result = { file: receiptFile, bytes: bytes.length, sha256: hash(bytes), passed: receipt.passed }; + if (failure) { + const error = new Error(`Physical quiet guard failed (${phase}); receipt=${receiptFile}; sha256=${result.sha256}`, { cause: failure }); + Object.assign(error, { receiptFile, receiptSha256: result.sha256 }); throw error; + } + return result; +} +export async function assertNoPreferredUiShadow(appRoot, statPath = lstat) { + const preferredIndex = path.join(appRoot, 'server/ui-dist/index.html'), preferredDirectory = path.dirname(preferredIndex); + try { await statPath(preferredIndex); assert.fail('Preferred server/ui-dist/index.html exists (including a symlink); ui/dist inventory cannot attest the selected UI'); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + try { assert.equal((await statPath(preferredDirectory)).isSymbolicLink(), false, 'Preferred server/ui-dist directory symlink prevents the absence guard'); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + return { checkedAtMs: Date.now(), preferredIndex, preferredIndexAbsent: true, preferredDirectorySymlink: false, + selectedUiRoot: path.join(appRoot, 'ui/dist'), + assumption: 'Preferred server/ui-dist index absent at preflight; inventoried ui/dist is the compiled fallback. No routing/build changes or future immutability claim.' }; +} + +async function checkFrozen(context, selectedVariant) { + await checkReleaseBytes(context); + for (const key of ['DATABASE_URL', 'PAPERCLIP_RUNNER_STATE_DIR', 'PAPERCLIP_INGESTION_SERVER_COMMAND', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR', + 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR', 'PAPERCLIP_INGESTION_BROWSER_CAPTURE', 'NODE_OPTIONS']) + assert.ok(!process.env[key], `Acceptance rejects ambient ${key}`); + const { release, plan } = context, actualInventories = {}; + for (const variant of plan.requiredVariants) { + const build = release.perVariantBuilds[variant], qualified = await validateVariantBuild(variant, build); + assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.revision, `${variant} SUT revision changed`); + assert.equal(execFileSync('git', ['status', '--short'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), '', `${variant} SUT is dirty`); + assert.equal((await getJson(path.join(build.appRoot, 'server/dist/build-info.json'))).commit, build.revision); + const binaryFile = path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'); + const native = qualified.receipt.nativeStageHashes.find(row => row.variant === variant); + const actualTarget = await realpath(path.join(build.appRoot, 'packages/paperclip-runner/runner/target')); + assert.equal(actualTarget, native.targetRealPath, `${variant} actual native target differs from qualified receipt`); + assert.equal(await realpath(binaryFile), path.join(actualTarget, 'release/paperclip-runnerd'), `${variant} native binary resolves outside its exact qualified target`); + assert.equal(hash(await readFile(binaryFile)), build.binarySha256, `${variant} actual native binary changed`); + for (const [file, object] of Object.entries(qualified.receipt.rustInputObjects)) + assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/${file}`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), object, `${variant} qualified Rust input changed: ${file}`); + assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/runner`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.rustSourceTree); + if (!selectedVariant || selectedVariant === variant) actualInventories[variant] = { + ...await validateActualInventory(build, qualified.manifest), uiServing: await assertNoPreferredUiShadow(build.appRoot), + }; + } + assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), release.harnessRevision); + assert.equal(execFileSync('git', ['status', '--short'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), ''); + assert.equal(hash(Buffer.concat(await Promise.all(protocolFiles.map(file => readFile(path.join(release.harnessRoot, 'tests/e2e', file)))))), release.protocolSha256); + for (const [file, expected] of [['scripts/compare-native-ingestion-browser.ts', release.comparatorSha256], ['scripts/native-ingestion-h5-inventory.mjs', release.inventoryHelperSha256], ['tests/e2e/fixtures/native-ingestion-h5.md', release.protocolDocumentSha256], ['scripts/native-ingestion-performance-gates.mjs', release.gatesSha256], ['pnpm-lock.yaml', release.lockSha256]]) + assert.equal(hash(await readFile(path.join(release.harnessRoot, file))), expected, `Frozen H5 support changed: ${file}`); + await checkPriorStages(context); + if (plan.noiseStage) await checkNoise(context); + return { checkedAtMs: Date.now(), releaseFileSha256: context.releaseBytesSha256, actualInventories }; +} + +async function execute(context, step) { + const index = ++context.commandIndex, startedAt = new Date().toISOString(), log = path.join(context.out, `${step.stage}.log`); + const command = { index, stage: step.stage, cwd: step.cwd, executable: process.execPath, args: step.args, environmentOverrides: step.env ?? {}, startedAt, log, + resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS, resourceWatchdogScope: 'Resource-only cap including startup/test/teardown; original SLA and H5 observation limits remain driver-owned' }; + context.commands.push(command); + const save = () => saveJson(path.join(context.out, `${context.plan.stage}-commands.json`), context.commands); + await save(); console.log(`START ${step.stage} ${startedAt}; log=${log}`); + const stream = (await open(log, 'wx')).createWriteStream(); + const child = spawn(process.execPath, step.args, { cwd: step.cwd, env: { ...process.env, ...step.env }, stdio: ['ignore', 'pipe', 'pipe'] }); + command.childPid = child.pid; await save(); + const interrupt = () => child.kill('SIGINT'), terminate = () => child.kill('SIGTERM'); + process.once('SIGINT', interrupt); process.once('SIGTERM', terminate); + let forcedStop; + const watchdog = setTimeout(() => { + command.watchdogExpired = true; child.kill('SIGTERM'); + forcedStop = setTimeout(() => { command.forcedDriverKill = true; child.kill('SIGKILL'); }, 30_000); + }, RESOURCE_WATCHDOG_MILLIS); + child.stdout.pipe(stream, { end: false }); child.stderr.pipe(stream, { end: false }); + let exit; + try { exit = await new Promise((resolve, reject) => { child.once('error', reject); child.once('close', (code, signal) => resolve({ code, signal })); }); } + finally { clearTimeout(watchdog); clearTimeout(forcedStop); process.off('SIGINT', interrupt); process.off('SIGTERM', terminate); await new Promise(resolve => stream.end(resolve)); } + Object.assign(command, { finishedAt: new Date().toISOString(), ...exit }); await save(); + console.log(`END ${step.stage} code=${exit.code} signal=${exit.signal ?? 'none'}`); + assert.equal(command.watchdogExpired, undefined, 'Resource watchdog expired; stop for physical cleanup, never accept or advance'); + assert.equal(exit.code, 0, `Stage ${step.stage} failed; retain raw files and perform physical cleanup before continuing`); +} + +async function freePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); }); + const port = server.address().port; await new Promise(resolve => server.close(resolve)); return port; +} +async function assertPortReleased(port) { + const server = net.createServer(); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); }); + await new Promise(resolve => server.close(resolve)); +} +async function extractEvidence(directory) { + const playwrightFile = path.join(directory, 'playwright.json'), report = await getJson(playwrightFile), attachments = []; + const visit = node => { if (Array.isArray(node)) for (const child of node) visit(child); else if (node && typeof node === 'object') { + if (Array.isArray(node.attachments)) attachments.push(...node.attachments.filter(value => value.name === 'native-ingestion-evidence')); + for (const [key, child] of Object.entries(node)) if (key !== 'attachments') visit(child); + } }; + visit(report); assert.equal(attachments.length, 1, 'Exactly one native-ingestion evidence attachment is required'); + const attachment = attachments[0]; + const evidence = attachment.body ? JSON.parse(Buffer.from(attachment.body, 'base64').toString()) : await getJson(path.resolve(directory, attachment.path)); + const file = path.join(directory, 'native-ingestion-evidence.json'); await saveJson(file, evidence, true); + return { file, evidence, playwrightFile }; +} + +async function armSidecar(directory, variant, build) { + const destination = path.join(directory, 'inventory'); await mkdir(destination); + const copies = {}; + for (const [key, sha] of [['manifestFile', 'manifestFileSha256'], ['inventorySummaryFile', 'inventorySummarySha256'], ['qualifiedReceiptFile', 'qualifiedReceiptSha256']]) { + const bytes = await readFile(build[key]); assert.equal(hash(bytes), build[sha], 'Per-arm provenance changed while copying'); + const file = path.join(destination, path.basename(build[key])); await writeFile(file, bytes, { flag: 'wx' }); copies[key] = path.relative(directory, file); + } + const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant, sourceRevision: build.revision, compiledBuildRevision: build.revision, + playwrightReportSha256: hash(await readFile(path.join(directory, 'playwright.json'))), ...copies, + manifestFileSha256: build.manifestFileSha256, aggregateSha256: build.aggregateSha256, + qualifiedReceiptSha256: build.qualifiedReceiptSha256, inventorySummarySha256: build.inventorySummarySha256 }; + const file = path.join(directory, 'arm-inventory.json'); await saveJson(file, sidecar, true); return file; +} + +async function armArtifactFiles(result) { + const sidecar = await getJson(result.inventoryFile), base = path.dirname(result.inventoryFile); + return [result.file, result.playwrightFile, result.inventoryFile, + ...['manifestFile', 'inventorySummaryFile', 'qualifiedReceiptFile'].map(key => path.resolve(base, sidecar[key])), + path.join(base, 'pre-arm-inventory-check.json'), path.join(base, 'physical-http-port-check.json'), + path.join(base, 'physical-quiet-pre.json'), path.join(base, 'physical-quiet-post.json')]; +} +export async function sealArmArtifacts(out, result) { + const canonicalRoot = await realpath(out), files = await armArtifactFiles(result), seen = new Set(), seals = []; + for (const file of files) { + absolute(file, 'raw arm artifact'); const canonical = await realpath(file); + assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Raw arm artifact resolves outside owned output'); + assert.equal(seen.has(canonical), false, 'Duplicate raw arm artifact'); seen.add(canonical); + const bytes = await readFile(file); seals.push({ file, bytes: bytes.length, sha256: hash(bytes) }); + } + return seals; +} +export async function verifyRawArtifactSeals(out, manifest) { + const canonicalRoot = await realpath(out); + for (const entry of manifest) for (const arm of ['baseline', 'candidate']) { + const seals = entry[`${arm}Artifacts`]; assert.ok(Array.isArray(seals) && seals.length === 10, 'Raw arm byte seals are incomplete'); + const seen = new Set(); + for (const seal of seals) { + absolute(seal.file, 'sealed arm artifact'); digest(seal.sha256, 'sealed arm artifact'); + assert.equal(seen.has(seal.file), false, 'Duplicate sealed arm artifact'); seen.add(seal.file); + assert.ok((await realpath(seal.file)).startsWith(`${canonicalRoot}${path.sep}`), 'Sealed arm artifact resolves outside output'); + const bytes = await readFile(seal.file); + assert.equal(bytes.length, seal.bytes, 'Raw arm artifact size changed after timing'); + assert.equal(hash(bytes), seal.sha256, 'Raw arm artifact bytes changed after timing'); + } + const expected = await armArtifactFiles({ file: entry[`${arm}File`], playwrightFile: entry[`${arm}PlaywrightFile`], inventoryFile: entry[`${arm}InventoryFile`] }); + assert.deepEqual([...seen].sort(), expected.sort(), 'Raw seals do not cover this exact evidence/PW/sidecar/inventory set'); + for (const [filename, phase] of [['physical-quiet-pre.json', 'pre-arm'], ['physical-quiet-post.json', 'post-arm']]) { + const receipt = await getJson(path.join(path.dirname(entry[`${arm}InventoryFile`]), filename)); + assert.equal(receipt.schema, QUIET_SCHEMA); assert.equal(receipt.passed, true); assert.equal(receipt.platform, 'linux'); + assert.equal(receipt.phase, phase); assert.equal(receipt.zombiePolicy, QUIET_ZOMBIE_POLICY); + } + } +} + +function physicalGuardReferences(manifest) { + return manifest.flatMap((entry, pair) => ['baseline', 'candidate'].map(arm => ({ pair: pair + 1, arm, + receipts: entry[`${arm}Artifacts`].filter(seal => /^physical-quiet-(pre|post)\.json$/.test(path.basename(seal.file))) }))); +} + +async function browserRun(context, armStage, variant) { + const preflight = await checkFrozen(context, variant); // ALL receipts; selected live artifact inventory before EVERY arm. + const { release } = context, build = release.perVariantBuilds[variant], directory = path.join(context.out, armStage); + await mkdir(directory, { recursive: false }); await saveJson(path.join(directory, 'pre-arm-inventory-check.json'), preflight, true); + await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-pre.json'), phase: 'pre-arm', armStage }); + const port = await freePort(); + let primaryError, inventoryFile, extracted; + try { + await execute(context, { stage: armStage, cwd: release.harnessRoot, + args: ['node_modules/@playwright/test/cli.js', 'test', '--config', 'tests/e2e/playwright-native-ingestion.config.ts'], + env: { PAPERCLIP_INGESTION_OBSERVATION_MODE: MODE, PAPERCLIP_RUNNER_BINARY: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'), + PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'), + PAPERCLIP_INGESTION_TRANSPORT: 'sandbox-remote-with-local-stop', PAPERCLIP_E2E_PORT: String(port), PAPERCLIP_INGESTION_SMOKE: '0', + PAPERCLIP_INGESTION_VARIANT: `${armStage}-${variant}`, PAPERCLIP_INGESTION_APP_ROOT: build.appRoot, PAPERCLIP_INGESTION_EVIDENCE_DIR: directory, + PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES: 'Independent frozen H5 driver install; each measured variant owns its separately qualified application/native build' } }); + } catch (error) { primaryError = error; } + try { + inventoryFile = await armSidecar(directory, variant, build); extracted = await extractEvidence(directory); + if (!primaryError) { + const { evidence } = extracted; + assert.equal(evidence.complete, true); assert.equal(evidence.observationComplete, true); assert.equal(evidence.observationMode, MODE); + assert.equal(evidence.workloadSha256, WORKLOAD_SHA256); assert.equal(evidence.protocolVersion, PROTOCOL); + assert.equal(evidence.transport, 'sandbox-remote-with-local-stop'); assert.equal(evidence.cleanup?.complete, true); assert.deepEqual(evidence.cleanup.errors, []); + assert.equal(evidence.measurementSha256, release.protocolSha256); assert.equal(evidence.harnessProvenance?.sourceRevision, release.harnessRevision); assert.equal(evidence.harnessProvenance?.sourceStatus, ''); + assert.equal(evidence.sourceRevision, build.revision); assert.equal(evidence.compiledBuildRevision, build.revision); + assert.equal(evidence.serverArtifact, 'compiled'); assert.equal(evidence.qualificationOnly, false); assert.equal(evidence.diagnosticOnly, false); assert.equal(evidence.performanceAcceptanceEligible, true); + assert.equal(evidence.browserRecording?.trace, 'off'); assert.equal(evidence.browserRecording?.video, 'off'); assert.equal(evidence.browserRecording?.screencast, false); + const loaded = evidence.cases.find(item => item.name === 'burst512'); assert.equal(loaded.runs, 100); assert.equal(loaded.participants, 100); assert.equal(loaded.durable.length, 100); + const { validateArmInventory } = await import(pathToFileURL(path.join(release.harnessRoot, 'scripts/native-ingestion-h5-inventory.mjs')).href); + await validateArmInventory(inventoryFile, { evidence, owningPlaywright: { reportSha256: hash(await readFile(extracted.playwrightFile)) } }); + } + } catch (error) { primaryError ??= error; } + try { await assertPortReleased(port); await saveJson(path.join(directory, 'physical-http-port-check.json'), { released: true, port, scope: 'Owned HTTP listener only; PG/Runner/provider physical cleanup is separately required before the next release' }, true); } + catch (error) { primaryError ??= error; } + try { await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-post.json'), phase: 'post-arm', armStage, + primaryFailureAlreadyPresent: Boolean(primaryError) }); } + catch (error) { + if (primaryError) console.error(`Secondary physical quiet failure; primary fixture failure is retained: ${error.message}`); + primaryError ??= error; + } // Failed guard receipt and SHA remain independent evidence; the fixture failure stays primary. + if (primaryError) throw primaryError; + const result = { file: extracted.file, playwrightFile: extracted.playwrightFile, inventoryFile }; + return { ...result, artifacts: await sealArmArtifacts(context.out, result) }; +} + +function comparatorArgs(context, plan, pairsFile, output, originalProof) { + const args = ['scripts/compare-native-ingestion-browser.ts', `--observation-mode=${MODE}`, `--phase=${plan.aa ? 'aa' : 'compare'}`, + `--expect=${plan.expectation}`, `--pairs=${pairsFile}`, `--output=${output}`]; + if (!plan.aa) args.push(`--candidate-policy=${plan.candidatePolicy}`, `--noise-report=${path.join(context.out, `${plan.noiseStage}.json`)}`, + ...(originalProof ? [`--original-h4d-proof=${originalProof.proofFile}`, `--original-h4d-inventory=${originalProof.inventoryFile}`, '--require-acceptance'] : ['--require-performance'])); + return args; +} + +async function browserSeries(context) { + const { plan, release } = context, manifest = [], file = path.join(context.out, `${plan.stage}-pairs.json`), comparisonFile = path.join(context.out, `${plan.stage}.json`); + await access(comparisonFile).then(() => assert.fail('Prior comparison output already exists'), error => { if (error.code !== 'ENOENT') throw error; }); + await saveJson(file, manifest, true); + for (let pair = 0; pair < 5; pair++) { + const entry = {}, arms = pair % 2 ? [['candidate', plan.candidate], ['baseline', plan.baseline]] : [['baseline', plan.baseline], ['candidate', plan.candidate]]; + for (const [arm, variant] of arms) { + const result = await browserRun(context, `${plan.stage}-pair${pair + 1}-${arm}`, variant); + entry[`${arm}File`] = result.file; entry[`${arm}PlaywrightFile`] = result.playwrightFile; entry[`${arm}InventoryFile`] = result.inventoryFile; entry[`${arm}Artifacts`] = result.artifacts; + } + manifest.push(entry); await saveJson(file, manifest); + } + await checkFrozen(context); // Same exact release/noise path and bytes pre-comparator. + await verifyRawArtifactSeals(context.out, manifest); + const manifestSha256 = hash(await readFile(file)); + await execute(context, { stage: `${plan.stage}-comparison`, cwd: release.harnessRoot, args: comparatorArgs(context, plan, file, comparisonFile) }); + assert.equal(hash(await readFile(file)), manifestSha256, 'Raw paired manifest changed during comparison'); + await verifyRawArtifactSeals(context.out, manifest); + const result = await getJson(comparisonFile); checkComparisonIdentity(result, context); + if (!plan.aa) assert.equal(result.performancePassed, true); + await saveJson(path.join(context.out, `${plan.stage}.completion.json`), { stage: plan.stage, complete: true, fullArms: 10, completedAtMs: Date.now(), + releaseFile: context.rootReleaseCopy, originalReleaseFile: context.releaseFile, releaseFileSha256: context.releaseBytesSha256, harnessPins: harnessFields(release), + perVariantBuilds: Object.fromEntries(plan.requiredVariants.map(variant => [variant, release.perVariantBuilds[variant]])), + manifestFile: file, manifestSha256, comparisonFile, comparisonSha256: hash(await readFile(comparisonFile)), + physicalGuards: physicalGuardReferences(manifest), + ...(plan.noiseStage ? { noiseReportFile: release.noiseReportFile, noiseReportSha256: release.noiseReportSha256 } : {}), + performancePassed: result.performancePassed ?? null, acceptancePassed: result.acceptancePassed ?? null, + finalOriginalH4dConfirmationPending: plan.candidatePolicy === 'final-b2' }, true); +} + +async function checkFinalizationInputs(context) { + const proof = context.release.originalH4dProof; + assert.ok(proof, 'One later unchanged H4d proof is required for both final comparisons'); + for (const [key, sha] of [['proofFile', 'proofSha256'], ['inventoryFile', 'inventorySha256']]) await hashedJson(proof[key], proof[sha], `Original H4d ${key}`); + const proofReport = await getJson(proof.proofFile), proofAttempts = []; + const visit = value => { + if (Array.isArray(value)) for (const child of value) visit(child); + else if (value && typeof value === 'object') { + if (Array.isArray(value.results) && Object.hasOwn(value, 'expectedStatus')) proofAttempts.push(...value.results); + for (const [key, child] of Object.entries(value)) if (key !== 'results') visit(child); + } + }; + visit(proofReport); assert.equal(proofAttempts.length, 1, 'One later unchanged original H4d attempt must confirm both stages'); + const proofStartedAtMs = Date.parse(proofAttempts[0].startTime); + assert.ok(Number.isFinite(proofStartedAtMs), 'Original H4d proof has no actual attempt start time'); + for (const stage of ['browser-b1-b2', 'browser-b0-b2']) { + const pin = context.release.completedSeries?.[stage]; assert.ok(pin, `Missing frozen completed ${stage}`); + assert.equal(pin.file, path.join(context.out, `${stage}.completion.json`)); + const completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value; + assert.equal(completion.stage, stage); assert.equal(completion.complete, true); assert.equal(completion.fullArms, 10); assert.equal(completion.performancePassed, true); + assert.ok(Number.isFinite(completion.completedAtMs) && completion.completedAtMs < proofStartedAtMs, 'Original H4d proof must run after BOTH timing stages complete'); + assert.deepEqual(completion.harnessPins, harnessFields(context.release)); + const plan = stagePlan(stage); + for (const variant of plan.requiredVariants) assert.deepEqual(completion.perVariantBuilds[variant], context.release.perVariantBuilds[variant], 'Finalization changed a measured qualified build'); + assert.equal(completion.manifestFile, path.join(context.out, `${stage}-pairs.json`)); + assert.equal(completion.comparisonFile, path.join(context.out, `${stage}.json`)); + assert.equal(hash(await readFile(completion.manifestFile)), completion.manifestSha256, 'Raw timing manifest changed before finalization'); + const manifest = await getJson(completion.manifestFile); assert.equal(manifest.length, 5); + await verifyRawArtifactSeals(context.out, manifest); + assert.deepEqual(completion.physicalGuards, physicalGuardReferences(manifest), 'Completion physical-guard receipts differ from sealed arm bytes'); + assert.equal(hash(await readFile(completion.comparisonFile)), completion.comparisonSha256, 'Preliminary performance assessment changed'); + assert.equal(completion.noiseReportFile, path.join(context.out, `${plan.noiseStage}.json`), 'Finalization rebound the frozen noise path'); + assert.equal(hash(await readFile(completion.noiseReportFile)), completion.noiseReportSha256, 'Frozen noise bytes changed before finalization'); + const originalRelease = await hashedJson(completion.releaseFile, completion.releaseFileSha256, `${stage} original release`); + assert.equal(originalRelease.value.noiseReportFile, completion.noiseReportFile); assert.equal(originalRelease.value.noiseReportSha256, completion.noiseReportSha256); + } +} + +async function finalizeSeries(context) { + await checkFrozen(context); await checkFinalizationInputs(context); + for (const stage of ['browser-b1-b2', 'browser-b0-b2']) { + await checkFrozen(context); await checkFinalizationInputs(context); + const pin = context.release.completedSeries[stage], completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value; + const output = path.join(context.out, `${stage}.final.json`); + await access(output).then(() => assert.fail('Final assessment output already exists'), error => { if (error.code !== 'ENOENT') throw error; }); + await execute(context, { stage: `${stage}-original-h4d-finalization`, cwd: context.release.harnessRoot, + args: comparatorArgs(context, stagePlan(stage), completion.manifestFile, output, context.release.originalH4dProof) }); + await checkFinalizationInputs(context); + assert.equal((await getJson(output)).acceptancePassed, true); + } +} + +async function main() { + const stage = option('stage'); + if (!process.argv.includes('--execute') && !process.argv.includes('--validate-release')) { + console.log(JSON.stringify({ draft: true, observationMode: MODE, protocolVersion: PROTOCOL, workloadSha256: WORKLOAD_SHA256, + stages: STAGES.map(stagePlan), fullArms: 50, finalization: stagePlan(FINALIZE_STAGE), + baselines: BASELINES, originalReceiptSha256: OLD_RECEIPT, resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS, + frozenH5: { revision: H5_REVISION, protocolSha256: H5_PROTOCOL_SHA256, comparatorSha256: H5_COMPARATOR_SHA256, inventoryHelperSha256: H5_INVENTORY_HELPER_SHA256, protocolDocumentSha256: H5_PROTOCOL_DOCUMENT_SHA256 }, + pendingPins: 'Future B2 has no descriptor until its actual new source and separately qualified native/application build are available. No H5 VM execution is released by this file.', + note: 'Plan only; no SUT/build/VM/network or arm starts. Root and independent peer review plus qualification release required.' }, null, 2)); return; + } + const out = option('output-root') ?? '/home/ubuntu/native-ingestion/evidence/h5/browser'; + const context = await loadRelease(option('release'), stage, out); + if (process.argv.includes('--validate-release') && !process.argv.includes('--execute')) { + console.log(JSON.stringify({ valid: true, stage, requiredVariants: context.plan.requiredVariants, + validationScope: 'Static immutable release/receipt/manifest/noise/stage-order bindings only; no live SUT/driver/build attestation and no arm execution', releaseFileSha256: context.releaseBytesSha256 })); return; + } + await checkFrozen(context); await mkdir(out, { recursive: true }); + // Keep the exact release bytes as a distinct artifact, so later stages never + // require an owner to retain an overwriteable release-input filename. + context.rootReleaseCopy = path.join(out, `${stage}-root-release.json`); + const releaseBytes = await readFile(context.releaseFile); + assert.equal(hash(releaseBytes), context.releaseBytesSha256, 'Root release changed before preservation'); + await writeFile(context.rootReleaseCopy, releaseBytes, { flag: 'wx' }); + await saveJson(path.join(out, `${stage}-release-record.json`), { ...context.release, releaseFile: context.releaseFile, + releaseFileSha256: context.releaseBytesSha256, recordedAt: new Date().toISOString() }, true); + if (context.plan.finalizationOnly) await finalizeSeries(context); else await browserSeries(context); +} +if (process.argv[1] && path.resolve(process.argv[1]) === selfFile) await main(); diff --git a/scripts/native-ingestion/h5b/sources.json b/scripts/native-ingestion/h5b/sources.json new file mode 100644 index 00000000000..0320936d492 --- /dev/null +++ b/scripts/native-ingestion/h5b/sources.json @@ -0,0 +1,73 @@ +{ + "schema": "paperclip.native-ingestion.h5b-helper-sources.v1", + "scope": "Exact reviewed task-helper source snapshots. Per-run owner receipts remain authoritative for executed file hashes. This packaging did not execute a new driver, application, qualification or performance arm.", + "frozenDriver": { + "tag": "native-ingestion-h5b", + "tagObject": "34498d7e09b61493e0f6703c354291c485836298", + "commit": "179fe6fa6da1b24ca08c1162301538d3e3788af6", + "lockSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8", + "measurementSha256": "452a5c8f0ad449b2db4ea3ca0bb6a6d19e8e6642eaac7b898caa55d1cedff40e", + "comparatorSha256": "0c3ecc2e992a84a010e0132e8efb129f6b9c02e478c2f45028235fc67b5e6168", + "inventoryHelperSha256": "76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9", + "protocolDocumentSha256": "60ae7052b9a58514d326109cdcf9f0c408d73acbff5d23be1fa92de1fcde8f54" + }, + "recordedLayout": { + "base": "/home/ubuntu/native-ingestion", + "driver": "harness-h5b", + "helperDirectory": "helpers", + "appDirectories": { + "B0": "aligned-baseline", + "B1": "aligned-cursors", + "B2": "aligned-batching" + } + }, + "recordedAppRevisions": { + "B0": "bac53ec3bfeaf6546e7aaeaee516487b30534654", + "B1": "07061a601cdc20603c8c7a46f9ddbad7ac5f5939", + "B2": "ca7cba171cd5198ed569d75012edc858041cbc44" + }, + "rawB2Role": "Unoptimized batching base only. No future Rust candidate or matching build is supplied by this package.", + "deliveryLockSha256AtPackaging": "7540075e96f9a93924a1e54775c17346c5760dd5d02902d83fb5228f918996ba", + "deliveryLockWasExecutedDriver": false, + "files": [ + { + "file": "native-ingestion-run-h5b-browser.mjs", + "originalTaskFile": "/private/tmp/native-ingestion-run-h5b-browser.mjs", + "bytes": 53930, + "sha256": "522d4fc718665ef484dec40ff56e463efe11b4ddf0c632c2d3f4811092d63fa6", + "byteIdenticalToReviewedOriginal": true, + "packagingChanges": [] + }, + { + "file": "native-ingestion-linux-h5b-qualification.py", + "originalTaskFile": "/private/tmp/native-ingestion-linux-h5b-qualification.py", + "bytes": 6904, + "sha256": "0d19a505ca860a7e8eb3975c4cef5a509b2ae449fab0d8d90dce830cb787219b", + "byteIdenticalToReviewedOriginal": true, + "packagingChanges": [] + }, + { + "file": "native-ingestion-h5b-linux-readiness.mjs", + "originalTaskFile": "/private/tmp/native-ingestion-h5b-linux-readiness.mjs", + "bytes": 3446, + "sha256": "5694d9ee274dfd0a1c77ac917c2bca84f6f0a858f4eabdd73e03241e02eda62e", + "byteIdenticalToReviewedOriginal": true, + "packagingChanges": [] + }, + { + "file": "native-ingestion-extract-h5b-qualification.py", + "originalTaskFile": "/private/tmp/native-ingestion-extract-h5b-qualification.py", + "bytes": 5704, + "sha256": "a052375a80b3abad4513cdf019325e83fed63c67d935f1dc2accba753ce589f0", + "byteIdenticalToReviewedOriginal": true, + "packagingChanges": [], + "reviewedSourceRevision": "Root-approved post-read strengthening; no measured driver/source/workload/deadline change", + "historicalSmallValidator": { + "sha256": "00930574798e3d56257a4913d82400ccd739b4bb52e9a24d5cb18a36892be970", + "bytes": 5445, + "usedBy": "Recorded SMALL independent post-read proof; source retained in the owning raw23-fileSMALLartifactmanifest", + "difference": "Packaged validator adds3existing-historypreludeProof assertions: run identity,1000losslessunique deltas, pre-epochverification/retainedhead. This packagedvalidator did not produce the SMALL result." + } + } + ] +} diff --git a/tests/e2e/fixtures/native-ingestion-h5.md b/tests/e2e/fixtures/native-ingestion-h5.md new file mode 100644 index 00000000000..fd0be3f4378 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-h5.md @@ -0,0 +1,225 @@ +# H5b independent-agent observation registration (prospective; no execution release) + +H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5` +(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`). +Its observed 120-second streaming failures remain failures. Aborted or profiled +attempts cannot be reused as complete timing arms. H5b uses one normal agent per concurrent remote participant. It repairs the +experiment topology while preserving observation of original deadline outcomes. +It does not optimize the product or turn earlier failures into passes. + +The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654` +and B1 `07061a601cdc20603c8c7a46f9ddbad7ac5f5939`. Raw B2 +`ca7cba171cd5198ed569d75012edc858041cbc44` is the unoptimized batching base, +not a qualified future performance candidate. A later candidate needs its actual +new immutable revision, normal build receipt and inventory; no candidate reference +or matching binary is presumed before those artifacts exist. The historical +a365/d8cd/914 application failures remain evidence of their recorded revisions. + +Select H5 explicitly with +`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5b-observe-original-sla-independent-agents`. +The new default H4d mode uses the same independent-agent geometry with the original +120/180/900-second deadlines. Observation mode uses the explicit H5b protocol. +Both modes hash the same nine source files, including the shared topology helper. +Their source revision and measurement bytes must match for later confirmation. +Old H4c/H5a protocols, geometry, arms and noise are ineligible for H5b. + +Run `node scripts/describe-native-ingestion-h5.mjs` to inspect the registration. +This is a dry run only: it starts no app, provider, database or browser and +accepts no execution arguments. It grants no runtime release. + +## Independent participant geometry and setup + +Create a pool of100 normal remote agents for the full fixture (three for smoke). +Each concurrent remote participant receives a distinct pool agent by index; the +pool is reused only across sequential cases. The100-run case is100 agents × one +run each. Agent POST, normal environment selection, managed files, permission +checks and maxConcurrentRuns50 configuration stay enabled. The separate local +Stop control agent and its normal local environment stay unchanged. + +Use exactly eight setup workers, each awaiting normal agent creation and the +existing60s lifecycle-ready poll. Keep index order despite completion order. +Attempt and settle the whole pool; any setup error fails the invocation before +case activation. Record every attempt, start/settlement time and status. No +partial pool, silent retry or reduced load can qualify. The existing per-case +READ ONLY authority snapshot must prove distinct agent_id values for every +remote participant, all drawn from the ready pool. No event polling is added. + +Agent creation occurs before the existing300s activation/readiness window starts. +That start point remains after the pool and background issue creation. All agent +setup remains inside the owning public900s default/1200s observation invocation; +no setup work is moved outside those timers or into measured epochs. The existing +waves of two provider activations, readiness polls and common source epoch stay +unchanged. All arms use the same geometry and eight-worker setup. Its actual +setup cost and100-agent full qualification are unverified until measured. + +This changes managed-directory contention and agent-list cardinality. A result +will describe independent-agent ingestion, not50 concurrent runs sharing one +agent. Do not generalize it to the retained old2×50 failure below. + +## Retained H5a negative result + +H5a7388 on application B0a365 used two normal agents with50 concurrent runs each. +All100 participants were admitted and clock-ready. The original120s poll failed +with54/512 visible markers; all512 became visible296.555s after that wait began. +All100 runs reached a terminal status55.234s later, with86 succeeded and14 failed. +Independent read-only PostgreSQL checks found all51,200 expected lossless deltas +and one accepted result for each run. This did not pass full qualification or A/A. + +Each of the14 failed runs recorded native_session_interrupted from a30s +agent-directory merge-lock timeout; each finalization retained retryable_failure +with native_workspace_sync_out_failed and an ACTIVE unreleased lease. Seven +failures mapped to each agent's shared file-sync root. Task workspace and lease +identities were distinct; no provider failure was recorded. The new geometry +removes that shared-agent contention from this experiment. It does not fix or +qualify the product's old2×50 concurrency limit or justify disabling file sync. + +A separate background presentation40P01 involved a run that succeeded. The +retained log shows its coordinator→issue→run wait, but lacks PostgreSQL's full +opposing lock cycle. Its cause and relationship to other contention are unproved. +It is not evidence that the14 run failures came from a database deadlock. All +historical failures remain separate; no failed H4c/H5a arm is reused as H5b noise. + +## Fixed work and observation boundaries + +Keep the five streaming event workloads and original local Stop case exactly as H4c: +paced 1×128 at25ms; burst 1×16; burst10×128; burst100×512; existing-history +1000 foreground prelude deltas plus measured10×128; local Stop1×512 at100ms, +clicked after at least three observed markers. Keep normal API/UI activation, +waves of two, shared300s readiness, actual GET+clock hellos, verified TLS, +closed READ ONLY REPEATABLE READ authority snapshot, source identities and +payloads, original nonawaited source-clock writes, and the existing2RAF observer. +No normalized-event polling channel, per-event HTTP, injected provider receipt, +raw recovery-policy change, diagnostic instrumentation or replacement app entry +is added. Fake-plugin streaming remains separate from real-cloud Stop proof. + +| Boundary | Original requirement | H5 observation | +| --- | --- | --- | +| Foreground markers | Original Playwright120s wait | Continue to an absolute600s ceiling from that same wait start | +| All terminal runs |180s after foreground completion (and Stop click) | Same180s after eventual foreground completion; hard failure at the ceiling | +| Stop acknowledgement and leases | Existing30s polls and ordering | Unchanged; full click→acknowledgement metric retained | +| Invocation | Original900s test budget | Actual1200s observation timeout, separately disclosed | + +The first120s poll is the original matcher. Only that exact known matcher +deadline can extend observation. Page/API errors, other assertions, terminal +failures and cleanup failures never become an allowed SLA miss. There is no +collection wait when markers are available. The600s ceiling is absolute from +the original wait start; continued observation uses only its remaining budget. + +Record the original wait start/deadline, actual last count and its observation +time, assessment time and error before continuing. Keep the original failure +after completion. The original120s clock starts before the common source epoch; +it is not silently rebased to epoch+120s. Derive a separate paint-deadline count +from retained2RAF timestamps; do not mislabel it as the original poll's count. + +The recorded `deadlineMs` is the nominal configured boundary and +`assessedAtMs` is the actual matcher return time. Frozen Playwright1.62.1 can +report its timeout before that nominal boundary when the next polling interval +would cross it. Its original matcher outcome, known timeout error and incomplete +count establish the original failure; timestamp ordering against the nominal +deadline does not change that outcome. Retain both raw times and continue only +until the unchanged absolute600s observation ceiling. + +If120s fails, original terminal processing was **not reached**. Later180s +terminal observations are separately labeled and cannot make the original +protocol pass. Keep epoch→all-terminal drainage unshortened. An additional +independent completion upper bound follows existing screenshot, reload, +persistence, clock and lease checks; it includes verification overhead and is +not an exact last-commit time. + +The owning public Playwright JSON result supplies duration, startTime, retry, +status, errors and configured timeout. Its observed total duration is an +independent conservative assessment of the original900s budget; hooks/teardown +can be treated differently by the original timeout. It is not an exact original +timer-expiration observation. A final candidate must have total duration<900s +in every arm and later pass one unchanged H4d default-mode full invocation on the +same final SUT revision and full qualified artifact inventory. + +## Complete evidence and eligibility + +H5 `observationComplete` means all six cases reached finite correct outcomes, +with every expected participant and foreground marker. It does not establish a +pass of the original deadlines. Every remote run must succeed in native mode, retain every normalized +delta's complete payload/identity/order, have exactly one independently read +`run.result.accepted`, and pass the existing reload/result/clock/lease checks. +The100-run case must independently verify all51200 deltas. Stop keeps its +original cancellation, durable acknowledgement, local journal, PID/group/provider +absence and lease checks; it must not acquire an accepted success result. + +An observation ceiling, failed/cancelled remote run, missing event/result, +unexpected API error, incomplete source clock, replay conflict or cleanup error +is ineligible. Preserve its raw partial evidence and censoring state. Never +substitute the ceiling as latency, compute a512-event tail from a visible prefix, +discard a slow arm, retry it into the sample, cancel background work to make a +pair finish, or hide work remaining after the measured interval. + +Each comparison-manifest entry contains `baselineFile`, `candidateFile`, +`baselinePlaywrightFile`, `candidatePlaywrightFile`, `baselineInventoryFile`, and +`candidateInventoryFile`. The owning Playwright result must contain exactly that +evidence attachment, one passed observation attempt, retry0, no errors, and the +actual1200s timeout. An inventory sidecar binds that report hash to the exact +gzip inventory bytes, full descriptor aggregate, summary and qualified build +receipt hashes, source/build revision, and reported serving/native fingerprints. +These frozen artifacts are evidence of the qualified build; the external owner +still checks the live checkout/build inventories before each arm. A sidecar does +not itself attest that the live filesystem stayed unchanged. + +| Role | Allowed original failures | Required result | +| --- | --- | --- | +| Fresh B0/B1 A/A arms | Known burst512 streaming120 miss; explicit conservative observed-duration900 miss | Complete finite correct observation; root-reviewed noise suitability | +| B0→B1 cursor-only candidate | Only an exact registered known miss also present in the paired baseline: burst512 streaming120 or conservative observed-duration900 | No additional deadline failures; latency/control nonregression plus separate incremental SQL evidence; never claim SLA fixed | +| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later H4d proof | + +Only burst512's streaming miss is registered as a known workload SLA failure. +New failures in other workloads require diagnosis. The existing Stop poll +ordering is preserved; no invented absolute click→ack30s requirement replaces it. + +## Stage order and acceptance + +Prospectively freeze **B0 A/A → B1 A/A → B0→B1 nonregression → B1→B2 → B0→B2**. +Each stage has exactly five fresh alternating pairs, separate disposable state +per arm, matched payloads/load/resource/cache policy, no reused baseline arms, +and no concurrent builds/profilers/benchmarks on the measurement VM. Root reviews +and freezes both complete A/A envelopes before any further product fix. Local +product implementation can then proceed while the unchanged B0→B1 stage runs; +the candidate VM build waits until that stage is quiet. + +Keep the existing per-metric maximum absolute paired A/A envelope, median gate, +and at least4/5 individual noise-exceeding loaded wins. Each loaded p95 +emission-to-visible, p95 intended-to-visible and terminal-drainage metric must +improve; other fixed latency/control metrics must not regress outside noise. +Do not infer p99 from sparse observations. There is no new fixed percentage A/A +cutoff: root must reject unsuitable noisy/nonstationary calibration explicitly. + +Use `--observation-mode=h5b-observe-original-sla-independent-agents` for the comparator. Candidate +comparisons also require `--candidate-policy=cursor-nonregression --expect=nonregression` +or `--candidate-policy=final-b2`. Final comparisons expose `performancePassed` +separately; `acceptancePassed` remains false until `--original-h4d-proof` and +`--original-h4d-inventory` establish the later fresh H4d default-deadline functional pass. +That H4d proof is full, unprofiled, retry0, default mode with original120/180/900s +bounds, on the same frozen H5b harness source, nine-file measurement bytes and +independent-agent geometry, and the exact final SUT revision/qualified inventory. +It uses a new disposable company and starts after all candidate arms. Old7d/H4c +proofs and H5a observations cannot confirm this geometry. Prior failed H4c attempts remain in the record. +Confirmation reuses the measured arms' authority, physical Stop, local lease and +remote lease binding/settlement checks. The original30s polls keep their existing +ordering; no absolute click-to-acknowledgement deadline is added. +Use the explicit H5 `--require-performance` gate for the two final comparison +stages while that later proof is pending; this requires every performance and +candidate original-SLA gate, but grants no final acceptance. After the one later +H4d default-mode qualification, recompute both reports against their unchanged paired +manifests with the proof and `--require-acceptance`. This re-evaluates retained +evidence; it neither reuses an arm in a different comparison nor reruns failures. + +Before any runtime release, review the exact H5 protocol/comparator/helper +hashes, registration dry run, outcome/eligibility tests, types and test listing. +Then qualify the changed observation path before fresh A/A. Each later stage +requires a separate root release pinning exact driver, build/inventory and +reviewed noise-report bytes; verify the noise hash before the first arm and +again before comparison. Product or workload changes require new prospective +provenance and affected requalification; do not loosen gates after results. + +Separate successful full-window diagnostics may later assess Node/PG CPU, +queue/backlog, delay and memory with identical capture hooks. Aborted profiles +remain diagnostic-only and cannot establish complete CPU/event costs. No CPU +requirement or profiling is added to this timing comparator. Local results do +not establish deployed improvement. diff --git a/tests/e2e/fixtures/native-ingestion-h5.mjs b/tests/e2e/fixtures/native-ingestion-h5.mjs new file mode 100644 index 00000000000..564511d7a26 --- /dev/null +++ b/tests/e2e/fixtures/native-ingestion-h5.mjs @@ -0,0 +1,254 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; + +export const H5_MODE = 'h5b-observe-original-sla-independent-agents'; +export const H5_PROTOCOL = 'v5b-independent-agents-complete-observation-original-sla'; +export const H4D_PROTOCOL = 'v4d-independent-agents-original-deadlines'; +export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant', + fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50, + poolReuse: 'only-across-sequential-cases' }); +export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654', + B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' }); +export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000, + streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }); +export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 }); +export const H5_WORKLOADS = Object.freeze([ + { name: 'paced', runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false }, + { name: 'burst16', runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false }, + { name: 'burst128', runs: 10, events: 128, intervalMs: 0, prelude: 0, stop: false }, + { name: 'burst512', runs: 100, events: 512, intervalMs: 0, prelude: 0, stop: false }, + { name: 'existing-history', runs: 10, events: 128, intervalMs: 0, prelude: 1000, stop: false }, + { name: 'operator-stop', runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true }, +]); +export const H5_REGISTRATION = Object.freeze({ mode: H5_MODE, protocolVersion: H5_PROTOCOL, + baselineRevisions: H5_BASELINE_REVISIONS, + topology: H5_TOPOLOGY, originalDeadlineProtocol: H4D_PROTOCOL, + originalBudgets: ORIGINAL_BUDGETS, observationLimits: H5_LIMITS, workloads: H5_WORKLOADS, + pairs: 5, order: 'baseline-first on even pair index; candidate-first on odd pair index', + stages: ['B0 A/A', 'B1 A/A', 'B0→B1 nonregression', 'B1→B2 improvement', 'B0→B2 improvement'], + knownBaselineMisses: ['burst512:streaming120', 'invocation:observed-total-duration900'], + baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of the H4d original deadlines', + cursorCandidateEligibility: 'Nonregression; only the paired baseline exact registered known misses may be inherited', + finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later H4d default-mode functional PASS on the same harness source and independent-agent geometry required', + noise: 'Fresh B0 and B1 A/A frozen before any further product implementation; no old abort/profile samples', + performance: 'Loaded p95 emission/intended-to-visible and terminal drainage improve beyond frozen noise in median and at least4/5 pairs; all other metrics nonregress', +}); +export const H5_WORKLOAD_SHA256 = createHash('sha256').update(JSON.stringify(H5_REGISTRATION)).digest('hex'); + +/** Agent setup settles in full before any case activation; its time remains + * inside the owning invocation and outside the existing admission300s window. */ +export async function prepareParticipantAgents({ count, createAndReady, now = Date.now, onOutcome = _value => {} }) { + assert.ok(Number.isSafeInteger(count) && count > 0 && count <= H5_TOPOLOGY.fullRemoteAgentPoolSize); + const agents = new Array(count), failures = []; + const outcome = { count, concurrency: H5_TOPOLOGY.agentSetupConcurrency, startedAtMs: now(), settledAtMs: null, + complete: false, attempts: [] }; + onOutcome(outcome); + let next = 0; + const workers = await Promise.allSettled(Array.from({ length: Math.min(count, outcome.concurrency) }, async () => { + for (;;) { + const index = next++; if (index >= count) return; + const attempt = { index, startedAtMs: now(), settledAtMs: null, status: 'pending', agentId: null }; + outcome.attempts.push(attempt); onOutcome(outcome); + try { + agents[index] = await createAndReady(index); + assert.ok(typeof agents[index]?.id === 'string', 'Missing ready participant agent'); + attempt.agentId = agents[index].id; attempt.status = 'fulfilled'; + } catch (error) { + failures.push(error); attempt.status = 'rejected'; attempt.error = String(error); + } finally { attempt.settledAtMs = now(); onOutcome(outcome); } + } + })); + for (const worker of workers) if (worker.status === 'rejected') failures.push(worker.reason); + if (!failures.length) try { assert.equal(new Set(agents.map(agent => agent.id)).size, count, 'Participant agents must be distinct'); } + catch (error) { failures.push(error); } + outcome.settledAtMs = now(); outcome.complete = failures.length === 0; onOutcome(outcome); + if (failures.length) throw new AggregateError(failures, 'Participant agent setup failed; no case may activate'); + return agents; +} + +export function assertParticipantSetup(report) { + assert.deepEqual(report.agentTopology, { ...H5_TOPOLOGY, remoteAgentPoolSize: report.qualificationOnly ? 3 : 100 }); + const ids = report.remote?.agentIds; + assert.equal(ids?.length, report.agentTopology.remoteAgentPoolSize, 'Incomplete independent participant agent pool'); + assert.equal(new Set(ids).size, ids.length, 'Duplicate participant agent pool identity'); + const setup = report.remote.agentSetup; + assert.equal(setup?.complete, true, 'Agent setup must settle successfully before any case'); + assert.equal(setup.count, ids.length); assert.equal(setup.concurrency, H5_TOPOLOGY.agentSetupConcurrency); + assert.ok(Number.isSafeInteger(report.bodyStartedAtMs) && report.bodyStartedAtMs > 0); + assert.ok(Number.isSafeInteger(setup.startedAtMs) && setup.startedAtMs >= report.bodyStartedAtMs); + assert.ok(Number.isSafeInteger(setup.settledAtMs) && setup.settledAtMs >= setup.startedAtMs); + assert.equal(setup.attempts?.length, ids.length); + assert.deepEqual(setup.attempts.map(attempt => attempt.index).sort((a, b) => a - b), ids.map((_id, index) => index)); + const boundaries = []; + for (const attempt of setup.attempts) { + assert.equal(attempt.status, 'fulfilled', 'Rejected agent setup cannot qualify'); + assert.equal(attempt.agentId, ids[attempt.index]); + assert.ok(Number.isSafeInteger(attempt.startedAtMs) && attempt.startedAtMs >= setup.startedAtMs); + assert.ok(Number.isSafeInteger(attempt.settledAtMs) && attempt.settledAtMs >= attempt.startedAtMs && attempt.settledAtMs <= setup.settledAtMs); + if (attempt.settledAtMs > attempt.startedAtMs) boundaries.push([attempt.startedAtMs, 1], [attempt.settledAtMs, -1]); + } + let active = 0; + for (const [_time, delta] of boundaries.sort((a, b) => a[0] - b[0] || a[1] - b[1])) { + active += delta; assert.ok(active <= setup.concurrency, 'Agent setup exceeded the fixed concurrency'); + } + for (const item of report.cases) assert.ok(item.activations?.[0]?.startedAtMs >= setup.settledAtMs, + 'Agent setup must finish before every case activation'); +} + +export function assertParticipantTopology(report, result) { + const ids = report.remote?.agentIds; + if (!result.stop) { + const rows = result.controllerAuthority?.rows; + assert.equal(rows?.length, result.runs, 'Missing participant authority rows'); + assert.equal(new Set(rows.map(row => row.agent_id)).size, result.runs, 'Concurrent participants must use distinct normal agents'); + assert.ok(rows.every(row => ids.includes(row.agent_id)), 'Participant authority is outside the normal agent pool'); + } +} + +export function h5Enabled(value) { + assert.ok(value === undefined || value === H5_MODE, 'Explicit supported H5 observation mode required'); + return value === H5_MODE; +} + +// Keep the original Playwright poll as the first wait. Only its known matcher +// deadline may extend observation; callback errors and other failures propagate. +export async function observeStreaming({ readCount, poll, target, now = Date.now, onOutcome = _value => {} }) { + const startedAtMs = now(); + let lastObservedCount = null, lastObservedAtMs = null, readError; + const read = async () => { + try { + const count = await readCount(); + assert.ok(Number.isSafeInteger(count) && count >= 0, 'Invalid visible marker count'); + lastObservedCount = count; lastObservedAtMs = now(); return count; + } catch (error) { readError = error; throw error; } + }; + let deadlineError; + try { await poll(read, ORIGINAL_BUDGETS.streamingMs, target); } + catch (error) { + if (readError) throw readError; + assert.ok(error?.matcherResult?.name === 'toBeGreaterThanOrEqual' && error.matcherResult.pass === false + && error.message.includes(`Timeout ${ORIGINAL_BUDGETS.streamingMs}ms exceeded while waiting on the predicate`), + 'Only the original streaming matcher deadline can extend observation'); + assert.ok(lastObservedCount !== null && lastObservedCount < target, 'Deadline needs an incomplete actual observation'); + deadlineError = error; + } + if (readError) throw readError; + // deadlineMs is the nominal configured boundary. Playwright can report its + // timeout earlier when its next polling interval would cross that boundary. + const original = { status: deadlineError ? 'failed' : 'passed', reason: deadlineError ? 'original_streaming_deadline' : null, + startedAtMs, deadlineMs: startedAtMs + ORIGINAL_BUDGETS.streamingMs, assessedAtMs: now(), + target, lastObservedCount, lastObservedAtMs, error: deadlineError?.message ?? null }; + const observation = { ceilingMs: H5_LIMITS.streamingMs, deadlineMs: startedAtMs + H5_LIMITS.streamingMs, + complete: false, censored: false, completedAtMs: null }; + const result = { original, observation }; + onOutcome(result); + if (deadlineError) { + try { + const remaining = observation.deadlineMs - now(); + assert.ok(remaining > 0, 'H5 streaming observation ceiling exceeded'); + await poll(read, remaining, target); + } catch (error) { observation.censored = true; onOutcome(result); throw readError ?? error; } + } + if (readError) throw readError; + observation.complete = true; observation.completedAtMs = now(); onOutcome(result); + return result; +} + +export function originalTerminal(streaming, terminal) { + return streaming.original.status === 'passed' + ? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null : 'original_terminal_deadline', + startedAtMs: terminal.startedAtMs, deadlineMs: terminal.deadlineMs, assessedAtMs: terminal.assessedAtMs } + : { status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null }; +} + +export function assessH5Arm(report, owner) { + assert.equal(report.protocolVersion, H5_PROTOCOL); + assert.equal(report.observationMode, H5_MODE); + assert.equal(report.observationComplete, true, 'H5 requires complete finite observation'); + assert.equal(report.complete, true); assert.equal(report.qualificationOnly, false); + assert.equal(report.diagnosticOnly, false); assert.equal(report.performanceAcceptanceEligible, true); + assert.equal(report.cleanup?.complete, true); assert.deepEqual(report.cleanup.errors, []); + assert.equal(report.workloadSha256, H5_WORKLOAD_SHA256, 'H5 registration changed'); + assert.deepEqual(report.observationLimits, H5_LIMITS); + assert.deepEqual(report.resourceBudgets, ORIGINAL_BUDGETS); + assert.equal(report.actualPlaywrightTimeoutMs, H5_LIMITS.fullInvocationMs); + assert.deepEqual(report.cases.map(item => item.name), H5_WORKLOADS.map(item => item.name), 'Complete fixed six-case observation required'); + assert.equal(owner.status, 'passed', 'Owning Playwright attempt must pass observation/correctness'); + assert.equal(owner.retry, 0, 'Retries cannot replace an H5 arm'); + assert.deepEqual(owner.errors, []); + assert.equal(owner.timeoutMs, H5_LIMITS.fullInvocationMs); + assert.ok(Number.isFinite(owner.durationMs) && owner.durationMs > 0 && owner.durationMs < H5_LIMITS.fullInvocationMs); + assert.ok(typeof owner.startedAt === 'string' && Number.isFinite(Date.parse(owner.startedAt))); + const misses = []; + for (const item of report.cases) { + const value = item.h5; + assert.ok(value, 'Missing H5 case outcomes'); + assert.equal(value.streaming.observation.complete, true); + assert.equal(value.streaming.observation.censored, false); + const streaming = value.streaming.original; + for (const field of ['startedAtMs', 'deadlineMs', 'assessedAtMs', 'lastObservedAtMs']) + assert.ok(Number.isSafeInteger(streaming[field]) && streaming[field] > 0, 'Missing raw original-deadline timing'); + assert.ok(['passed', 'failed'].includes(streaming.status)); + assert.equal(streaming.deadlineMs, streaming.startedAtMs + ORIGINAL_BUDGETS.streamingMs); + assert.equal(streaming.target, item.stop ? 3 : item.events); + assert.ok(Number.isSafeInteger(streaming.lastObservedCount) && streaming.lastObservedCount >= 0); + assert.ok(streaming.lastObservedAtMs >= streaming.startedAtMs && streaming.lastObservedAtMs <= streaming.assessedAtMs); + if (streaming.status === 'failed') { + assert.equal(streaming.reason, 'original_streaming_deadline'); + assert.ok(streaming.lastObservedCount < streaming.target); + // The original matcher outcome is authoritative; assessedAtMs retains + // the actual return time even when it precedes the nominal deadlineMs. + assert.equal(item.stop, false, 'Control readiness cannot inherit a streaming SLA miss'); + assert.ok(streaming.error.includes('Timeout 120000ms exceeded while waiting on the predicate')); + assert.equal(value.terminalOriginal.status, 'not-reached'); + assert.equal(value.terminalOriginal.reason, 'original_streaming_deadline'); + misses.push(`${item.name}:streaming120`); + } else { + assert.ok(streaming.lastObservedCount >= streaming.target); + assert.equal(streaming.reason, null); + assert.equal(streaming.error, null); + assert.equal(value.terminalOriginal.status, 'passed'); + } + assert.equal(value.terminalObservation.complete, true, 'H5 terminal observation must drain'); + assert.equal(value.terminalObservation.censored, false); + assert.equal(value.terminalObservation.deadlineMs, value.terminalObservation.startedAtMs + H5_LIMITS.terminalDrainageMs); + assert.equal(value.terminalObservation.assessedAtMs, item.settledAtMs); + assert.equal(value.terminalObservation.observedRuns, item.runs); assert.equal(value.terminalObservation.terminalRuns, item.runs); + assert.ok(value.terminalObservation.lastObservedAtMs >= value.terminalObservation.startedAtMs + && value.terminalObservation.lastObservedAtMs <= value.terminalObservation.assessedAtMs); + assert.deepEqual(value.terminalObservation.lastStatuses.map(row => row.runId).sort(), [...item.runIds].sort()); + assert.ok(value.terminalObservation.lastStatuses.every(row => row.status === (item.stop ? 'cancelled' : 'succeeded'))); + assert.equal(value.streaming.observation.ceilingMs, H5_LIMITS.streamingMs); + assert.equal(value.streaming.observation.deadlineMs, streaming.startedAtMs + H5_LIMITS.streamingMs); + assert.ok(value.streaming.observation.completedAtMs >= streaming.assessedAtMs + && value.streaming.observation.completedAtMs <= value.streaming.observation.deadlineMs); + assert.ok(value.terminalObservation.startedAtMs >= value.streaming.observation.completedAtMs); + assert.ok(item.settledAtMs <= value.terminalObservation.deadlineMs); + assert.deepEqual(value.terminalOriginal, originalTerminal(value.streaming, value.terminalObservation)); + assert.equal(value.visibleCountByOriginalPaintDeadline, item.visible.filter(marker => marker.visibleMs <= streaming.deadlineMs).length); + assert.deepEqual(item.terminalStatuses.map(row => row.runId).sort(), [...item.runIds].sort()); + assert.ok(item.terminalStatuses.every(row => row.runtimeMode === 'native' && row.status === (item.stop ? 'cancelled' : 'succeeded')), + 'Failed terminal runs are not successful drainage'); + assert.ok(item.durable.every(row => row.resultAcceptedEvents === (item.stop ? 0 : 1)), 'Every completed run needs its independently persisted result'); + assert.ok(value.independentDrain.verifiedAtMs >= item.settledAtMs); + assert.equal(value.independentDrain.elapsedFromEpochMs, value.independentDrain.verifiedAtMs - item.epochMs); + assert.equal(value.independentDrain.deltaCount, item.durable.reduce((sum, row) => sum + row.individualDeltas, 0)); + assert.equal(value.independentDrain.resultAcceptedCount, item.stop ? 0 : item.runs); + assert.equal(value.independentDrain.clockStreamsClosed, true); + } + if (owner.durationMs >= ORIGINAL_BUDGETS.fullInvocationMs) misses.push('invocation:observed-total-duration900'); + return { originalSlaPassed: misses.length === 0, misses, + original900budget: { status: owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs ? 'passed' : 'failed', + assessedAgainst: 'observed owning Playwright total duration', durationMs: owner.durationMs, + limitMs: ORIGINAL_BUDGETS.fullInvocationMs, + scope: 'Independent conservative gate; total duration can include hooks/teardown that the original timeout treats differently; not an exact original timer-expiration fact' } }; +} + +export function requireH5Candidate(policy, baseline, candidate) { + if (policy === 'final-b2') assert.deepEqual(candidate.misses, [], 'Final B2 must pass every original SLA in every arm'); + else if (policy === 'cursor-nonregression') { + const allowed = new Set(H5_REGISTRATION.knownBaselineMisses); + assert.ok(candidate.misses.every(miss => allowed.has(miss) && baseline.misses.includes(miss)), + 'Cursor-only candidate may inherit only the paired baseline exact registered known misses'); + } else throw new Error('Explicit final-b2 or cursor-nonregression candidate policy required'); +} diff --git a/tests/e2e/native-event-ingestion.journey.ts b/tests/e2e/native-event-ingestion.journey.ts index bce7653d9db..14eb1474963 100644 --- a/tests/e2e/native-event-ingestion.journey.ts +++ b/tests/e2e/native-event-ingestion.journey.ts @@ -7,6 +7,8 @@ import { hostname } from "node:os"; import path from "node:path"; import { expect, test } from "@playwright/test"; import { json } from "./agent-chat.shared"; +import { h5Enabled, H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS, + observeStreaming, originalTerminal, prepareParticipantAgents, assertParticipantTopology } from "./fixtures/native-ingestion-h5.mjs"; type SeenMarker = { markerCode: string; index: number; visibleMs: number }; type Emission = { nonce: string; markerCode: string; index: number; intendedMs: number; emittedMs: number; receivedAtMs: number }; @@ -23,6 +25,7 @@ type SetupActivation = { index: number; issueId: string; token: string; waveInde type SetupWave = { index: number; indices: number[]; startedAtMs: number; settledAtMs?: number; statuses?: Array<"fulfilled" | "rejected"> }; const smoke = process.env.PAPERCLIP_INGESTION_SMOKE === "1"; +const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE); const transport = process.env.PAPERCLIP_INGESTION_TRANSPORT ?? "sandbox-remote-with-local-stop"; const providerReadinessMs = 300_000; const setupActivationConcurrency = 2; @@ -150,6 +153,7 @@ const processAbsent = (pid: number) => { }; test("native ingestion reaches the shipped UI promptly and survives reload", async ({ page, request }, info) => { + const bodyStartedAtMs = Date.now(); const pending = new Map>(); const clocks = new Map(); let markerCounter = 0; @@ -271,10 +275,14 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video), screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" }; const report: Record = { - protocolVersion: "v4-remote-streaming-original-local-stop-clock-stream", transport, + protocolVersion: h5 ? H5_PROTOCOL : H4D_PROTOCOL, transport, bodyStartedAtMs, + agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: Math.max(...cases.map(item => item.runs)) }, + ...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS, + observationComplete: false, actualPlaywrightTimeoutMs: info.timeout, + originalInvocationBudgetAssessment: "Requires independent owning Playwright observed total duration; conservative original900 assessment, not exact timeout expiration" } : {}), transportPolicy: { streaming: "sandbox-remote", operatorStop: "local", realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" }, - resourceBudgets: { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000, + resourceBudgets: h5 ? ORIGINAL_BUDGETS : { providerReadinessMs, fullInvocationMs: info.timeout, streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 }, admissionProtocol: "two-at-a-time-provider-and-clock-readiness-before-common-epoch", setupActivationConcurrency, @@ -307,7 +315,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy report.fixtureSha256 = createHash("sha256").update(await readFile(path.resolve(import.meta.dirname, "fixtures/in-feed-codex.mjs"))).digest("hex"); const protocolFiles = ["native-event-ingestion.journey.ts", "playwright-native-ingestion.config.ts", "fixtures/in-feed-codex.mjs", "fixtures/native-ingestion-bin/codex", "fixtures/composer-stop-codex-loader.mjs", "fixtures/native-ingestion-bootstrap.mjs", - "fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs"]; + "fixtures/native-ingestion-remote-server.mjs", "fixtures/native-ingestion-protocol-qualification.mjs", + "fixtures/native-ingestion-h5.mjs"]; const protocolSources = await Promise.all(protocolFiles.map(async file => ({ file, bytes: await readFile(path.resolve(import.meta.dirname, file)) }))); report.measurementSha256 = createHash("sha256").update(Buffer.concat(protocolSources.map(source => source.bytes))).digest("hex"); const harnessRoot = path.resolve(import.meta.dirname, "../.."); @@ -349,9 +358,10 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy expect(localEnvironments).toHaveLength(1); const localEnvironment = localEnvironments[0]; expect(localEnvironment.driver).toBe("local"); - const agents: Array<{ id: string; name: string }> = []; - const agentCount = Math.ceil(Math.max(...cases.map(item => item.runs)) / 50); - for (let index = 0; index < agentCount; index++) { + const agentCount = Math.max(...cases.map(item => item.runs)); + const agents = await prepareParticipantAgents({ count: agentCount, + onOutcome: (value: unknown) => { (report.remote as any).agentSetup = value; }, + createAndReady: async (index: number) => { const agent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: { name: `Ingestion ${index + 1}`, role: "engineer", adapterType: "paperclip_runner", defaultEnvironmentId: remoteEnvironment.id, @@ -361,8 +371,8 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy } })); await expect.poll(async () => (await json(await request.get(`/api/agents/${agent.id}`))).lifecycleState, { timeout: 60_000 }).toBe("ready"); - agents.push(agent); - } + return agent; + } }) as Array<{ id: string; name: string }>; (report.remote as any).agentIds = agents.map(agent => agent.id); const controlAgent = await json(await request.post(`/api/companies/${company.id}/agents`, { timeout: 120_000, data: { name: "Ingestion local Stop control", role: "engineer", adapterType: "paperclip_runner", @@ -429,7 +439,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy register(backgroundToken, backgroundCode); const background = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: `Ingestion background ${key} ${index}`, description: prompt(backgroundToken, backgroundCode, 0), - status: "backlog", assigneeAgentId: caseAgents[Math.floor(index / 50)].id, + status: "backlog", assigneeAgentId: caseAgents[index].id, } })); issueIds.push(background.id); issuedTokens.set(background.id, backgroundToken); @@ -508,6 +518,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy appRoot, companyId: company.id, environmentId: caseEnvironment.id, issueIds, transport: caseTransport, }); active.controllerAuthority = controllerAuthority; + assertParticipantTopology(report, { ...workload, stop, controllerAuthority }); const providerClocks = controllerAuthority.rows.map((row: any) => { const token = issuedTokens.get(row.issue_id)!, stream = state.streams.get(token)!; return { runId: row.run_id, nonce: token, launcher: stream.launcher, providerPid: stream.providerPid }; @@ -529,7 +540,12 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy active.participants = participants; for (const response of pending.get(key)!.values()) response.writeHead(200, { "Content-Type": "application/json" }).end(JSON.stringify({ epochMs })); const observed = () => page.evaluate(code => (window as unknown as { nativeIngestionVisible: SeenMarker[] }).nativeIngestionVisible.filter(item => item.markerCode === code), markerCode); - await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events); + let streamingObservation: Awaited> | undefined; + if (h5) streamingObservation = await observeStreaming({ readCount: async () => (await observed()).length, + target: stop ? 3 : workload.events, + poll: (read: () => Promise, timeout: number, target: number) => expect.poll(read, { timeout }).toBeGreaterThanOrEqual(target), + onOutcome: (value: unknown) => { active.streamingObservation = value; } }); + else await expect.poll(async () => (await observed()).length, { timeout: 120_000 }).toBeGreaterThanOrEqual(stop ? 3 : workload.events); let stopRequestedAt: number | undefined; if (stop) { const button = page.getByRole("button", { name: "Stop", exact: true }); @@ -539,11 +555,25 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy } active.phase = "terminal-drainage"; let runs: any[] = []; - await expect.poll(async () => { + const terminalStartedAtMs = h5 ? Date.now() : undefined; + const terminalObservation = h5 ? { startedAtMs: terminalStartedAtMs!, deadlineMs: terminalStartedAtMs! + H5_LIMITS.terminalDrainageMs, + assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0, + lastStatuses: [] as Array<{ runId: string; status: string }>, complete: false, censored: false } : undefined; + if (h5) active.terminalObservation = terminalObservation; + try { await expect.poll(async () => { runs = (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId)); + if (terminalObservation) Object.assign(terminalObservation, { lastObservedAtMs: Date.now(), observedRuns: runs.length, + terminalRuns: runs.filter(run => ["succeeded", "cancelled", "failed"].includes(run.status)).length, + lastStatuses: runs.map(run => ({ runId: run.id, status: run.status })) }); return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status)); - }, { timeout: 180_000 }).toBe(true); + }, { timeout: 180_000 }).toBe(true); } + catch (error) { + if (terminalObservation) { terminalObservation.assessedAtMs = Date.now(); terminalObservation.censored = true; + active.terminalOriginal = originalTerminal(streamingObservation!, terminalObservation); } + throw error; + } const settledAt = Date.now(), seenMarkers = await observed(); + if (terminalObservation) { terminalObservation.assessedAtMs = settledAt; terminalObservation.complete = true; } active.settledAt = settledAt; active.seenMarkers = seenMarkers; const details = []; for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`))); @@ -618,6 +648,11 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const interChunkGaps = { paintCount: paintTimes.length, count, medianMs, maxMs }; const clockEvidence: Array> = []; const result = { ...workload, stop, transport: caseTransport, environmentId: caseEnvironment.id, + ...(h5 ? { h5: { streaming: streamingObservation, terminalObservation, + terminalOriginal: originalTerminal(streamingObservation!, terminalObservation!), + visibleCountByOriginalPaintDeadline: seenMarkers.filter(marker => marker.visibleMs <= streamingObservation!.original.deadlineMs).length, + deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : {}), + ...(h5 ? { terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status })) } : {}), setupActivationConcurrency, setupReadinessStartedAtMs, readinessDeadlineMs, allReadyObservedAtMs, activations, activationWaves, nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt, stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps, @@ -648,16 +683,19 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy const code = state.streams.get(token)!.markerCode; expect(token).toBeTruthy(); const deltas: Array<{ index: number; identity: string }> = []; - let afterSeq = 0; + let afterSeq = 0, resultAcceptedEvents = 0; while (true) { const rows = await json(await request.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)); if (!rows.length) break; for (const row of rows) { const event = row.payload?.prpEvent; + if (h5 && (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted")) resultAcceptedEvents++; if (event?.eventType !== "item.delta") continue; for (const match of String(event.payload?.text ?? "").matchAll(new RegExp(`\\[${code}([a-z0-9]{1,3})\\]`, "g"))) { const index = Number.parseInt(match[1], 36); expect(event.payload.text, "Every durable normalized delta retains its complete compact payload").toBe(`[${code}${index.toString(36)}] `); + if (h5) { expect(event.runId).toBe(run.id); expect(event.sourceInstanceId).toBeTruthy(); + expect(event.sourceEventId).toBeTruthy(); expect(Number.isSafeInteger(event.sourceSeq) && event.sourceSeq > 0).toBe(true); } deltas.push({ index, identity: `${event.sourceInstanceId}:${event.sourceEventId}:${event.sourceSeq}` }); } } @@ -674,11 +712,21 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy expect(new Set(deltas.map(item => item.identity)).size).toBe(deltas.length); expect(deltas.map(item => item.index)).toEqual(Array.from({ length: stop ? deltas.length : workload.events }, (_, index) => index)); if (stop) for (const item of visible) expect(deltas.some(delta => delta.index === item.index)).toBe(true); - durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, matchedSourceClocks: deltas.length, ordered: true }); + if (h5) expect(resultAcceptedEvents).toBe(stop ? 0 : 1); + durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length, + matchedSourceClocks: deltas.length, ordered: true, ...(h5 ? { resultAcceptedEvents } : {}) }); } await expect.poll(() => [...state.streams.values()].every(stream => stream.ended || stream.aborted), { timeout: 30_000 }).toBe(true); expect(state.errors).toEqual([]); expect([...state.streams.values()].every(stream => !stream.error)).toBe(true); + if (h5) { + const verifiedAtMs = Date.now(); + Object.assign(result.h5!, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs, + deltaCount: durable.reduce((count, row) => count + row.individualDeltas, 0), + resultAcceptedCount: durable.reduce((count, row) => count + (row.resultAcceptedEvents ?? 0), 0), + clockStreamsClosed: true, + scope: "Independent completion upper bound after existing reload/API/clock/lease checks; includes verification and screenshot overhead, not exact last-commit time" } }); + } clockEvidence.push(...[...state.streams.values()].map(stream => ({ nonce: stream.nonce, markerCode: stream.markerCode, launcher: stream.launcher, providerPid: stream.providerPid, hello: stream.hello, ended: stream.ended, aborted: stream.aborted, error: stream.error, partialBytes: stream.partialBytes, @@ -691,6 +739,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy for (const workload of cases) await perform(workload); await perform({ name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0 }, true); report.complete = true; + if (h5) report.observationComplete = true; } catch (error) { report.primaryFailure = { name: error instanceof Error ? error.name : "UnknownError", message: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined }; @@ -728,7 +777,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy await attemptCleanup("restore-experimental-settings", async () => json(await request.patch("/api/instance/settings/experimental", { data: { enableNativeRunner: original.enableNativeRunner, enableWorktreeRunExecution: original.enableWorktreeRunExecution } }))); report.cleanup = { errors: cleanupErrors, complete: cleanupErrors.length === 0 }; - if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; } + if (cleanupErrors.length) { report.complete = false; report.performanceAcceptanceEligible = false; + if (h5) report.observationComplete = false; } + if (h5) report.bodyFinishedAtMs = Date.now(); await info.attach("native-ingestion-evidence", { body: JSON.stringify(report, null, 2), contentType: "application/json" }); if (!report.primaryFailure) expect(cleanupErrors, "API cleanup must succeed before accepting an invocation").toEqual([]); } diff --git a/tests/e2e/playwright-native-ingestion.config.ts b/tests/e2e/playwright-native-ingestion.config.ts index 47c7ce3276f..083380b1ca5 100644 --- a/tests/e2e/playwright-native-ingestion.config.ts +++ b/tests/e2e/playwright-native-ingestion.config.ts @@ -3,9 +3,11 @@ import os from "node:os"; import path from "node:path"; import { defineConfig, type ReporterDescription } from "@playwright/test"; import base from "./playwright.config"; +import { h5Enabled, H5_LIMITS } from "./fixtures/native-ingestion-h5.mjs"; const server = base.webServer as Exclude; const evidence = process.env.PAPERCLIP_INGESTION_EVIDENCE_DIR; +const h5 = h5Enabled(process.env.PAPERCLIP_INGESTION_OBSERVATION_MODE); const browserRecording = Boolean(process.env.PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR) || process.env.PAPERCLIP_INGESTION_BROWSER_CAPTURE === "1"; process.env.PAPERCLIP_INGESTION_BROWSER_RECORDING = browserRecording ? "on" : "off"; const appRoot = path.resolve(process.env.PAPERCLIP_INGESTION_APP_ROOT ?? path.resolve(import.meta.dirname, "../..")); @@ -33,7 +35,7 @@ const providerHome = fs.mkdtempSync(path.join(os.tmpdir(), "ingestion-provider-" export default defineConfig({ ...base, testMatch: "native-event-ingestion.journey.ts", - timeout: 900_000, + timeout: h5 ? H5_LIMITS.fullInvocationMs : 900_000, ...(evidence ? { outputDir: path.join(evidence, "test-results"), reporter: [["list"], ["json", { outputFile: path.join(evidence, "playwright.json") }]] as ReporterDescription[] } : {}), use: { ...base.use, trace: browserRecording ? "on" : "off", video: "off", actionTimeout: 20_000 },