mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: retain frozen H5c settlement and acceptance helpers
Preserve the exact reviewed driver and helper source, qualified build pins, and reproduction commands. Keep original performance deadlines and retain recovered errors when determining eligibility and baseline retry noise.
This commit is contained in:
1 parent
652e4aebff
commit
6757908c19
14 files changed
+1660
-106
No files matched your search
@@ -3,8 +3,8 @@ import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { noiseExceedingPairedWins } from "./native-ingestion-performance-gates.mjs";
|
||||
import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS,
|
||||
assessH5Arm, requireH5Candidate, assertParticipantSetup, assertParticipantTopology } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { H5_MODE, H5_PROTOCOL, H4E_PROTOCOL, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS, ORIGINAL_BUDGETS,
|
||||
assessH5Arm, requireH5Candidate, assertParticipantSetup, assertParticipantTopology, assertSettlementObservation, assertRecordedFailures } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { validateArmInventory } from "./native-ingestion-h5-inventory.mjs";
|
||||
|
||||
// Each manifest entry points at native-ingestion-evidence JSON attachments
|
||||
@@ -18,6 +18,7 @@ const h5 = arg("observation-mode") === H5_MODE;
|
||||
assert.ok(arg("observation-mode") === undefined || h5, "Unknown observation mode");
|
||||
const candidatePolicy = arg("candidate-policy");
|
||||
assert.ok(!arg("original-h4c-proof") && !arg("original-h4c-inventory"), "Old H4c proof arguments cannot confirm the new independent-agent geometry");
|
||||
assert.ok(!arg("original-h4d-proof") && !arg("original-h4d-inventory"), "Old H4d proof arguments use the premature settlement observer");
|
||||
assert.ok(["aa", "compare"].includes(phase));
|
||||
assert.ok(["improvement", "nonregression"].includes(expectation));
|
||||
const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Array<{ baselineFile: string; candidateFile: string;
|
||||
@@ -154,6 +155,8 @@ function assertCaseAuthority(report: any, result: any) {
|
||||
}
|
||||
|
||||
function assertCaseSettlement(report: any, result: any, validated = assertCaseAuthority(report, result)) {
|
||||
assertSettlementObservation(result.h5?.terminalObservation ?? result.terminalObservation, result);
|
||||
assertRecordedFailures(result.recordedFailures, result, report.companyId);
|
||||
const remote = report.remote, local = report.local;
|
||||
const { authority, authorityByRun } = validated;
|
||||
if (result.stop) {
|
||||
@@ -229,7 +232,7 @@ function assertCaseSettlement(report: any, result: any, validated = assertCaseAu
|
||||
|
||||
for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const) {
|
||||
const report = pair[arm];
|
||||
assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : H4D_PROTOCOL,
|
||||
assert.equal(report.protocolVersion, h5 ? H5_PROTOCOL : H4E_PROTOCOL,
|
||||
"Fresh browser acceptance requires the independent-agent observation or default-deadline protocol");
|
||||
assert.equal(report.transport, "sandbox-remote-with-local-stop", "V4 requires remote streaming and the original local Stop transport");
|
||||
assert.deepEqual(report.transportPolicy, { streaming: "sandbox-remote", operatorStop: "local",
|
||||
@@ -401,25 +404,25 @@ for (const pair of pairs) for (const arm of ["baseline", "candidate"] as const)
|
||||
for (const field of ["runs", "events", "intervalMs", "prelude", "stop"]) assert.equal(result[field], reference[field], `Workload changed: ${name}.${field}`);
|
||||
}
|
||||
}
|
||||
let originalH4dConfirmation: Record<string, any> | undefined;
|
||||
let originalH4eConfirmation: Record<string, any> | undefined;
|
||||
if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
originalH4dConfirmation = { required: true, passed: false, reason: "Later H4d default-mode full functional proof on the same source and geometry is pending" };
|
||||
if (arg("original-h4d-proof")) {
|
||||
originalH4eConfirmation = { required: true, passed: false, reason: "Later H4e default-mode full functional proof on the same source and geometry is pending" };
|
||||
if (arg("original-h4e-proof")) {
|
||||
try {
|
||||
assert.ok(arg("original-h4d-inventory"), "H4d original-deadline proof requires its full artifact inventory sidecar");
|
||||
const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4d-proof")!));
|
||||
assert.ok(arg("original-h4e-inventory"), "H4e original-deadline proof requires its full artifact inventory sidecar");
|
||||
const { evidence: proof, owner } = await owningResult(path.resolve(arg("original-h4e-proof")!));
|
||||
assert.equal(owner.status, "passed"); assert.equal(owner.retry, 0); assert.deepEqual(owner.errors, []);
|
||||
assert.equal(owner.timeoutMs, ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.ok(owner.durationMs > 0 && owner.durationMs < ORIGINAL_BUDGETS.fullInvocationMs);
|
||||
assert.equal(proof.protocolVersion, H4D_PROTOCOL, "Old H4c geometry cannot confirm H5b");
|
||||
assert.equal(proof.protocolVersion, H4E_PROTOCOL, "Old H4c geometry cannot confirm H5c");
|
||||
assertParticipantSetup(proof);
|
||||
assert.deepEqual(proof.agentTopology, first.candidate.agentTopology);
|
||||
assert.equal(proof.observationMode, undefined, "Confirmation must use H4d default deadlines");
|
||||
assert.equal(proof.harnessProvenance?.sourceRevision, first.candidate.harnessProvenance.sourceRevision, "H4d confirmation must use the same frozen harness source");
|
||||
assert.equal(proof.observationMode, undefined, "Confirmation must use H4e default deadlines");
|
||||
assert.equal(proof.harnessProvenance?.sourceRevision, first.candidate.harnessProvenance.sourceRevision, "H4e confirmation must use the same frozen harness source");
|
||||
assert.deepEqual(proof.harnessProvenance.sourceHashes, first.candidate.harnessProvenance.sourceHashes);
|
||||
assert.equal(proof.harnessProvenance.driverLockSha256, first.candidate.harnessProvenance.driverLockSha256);
|
||||
assert.equal(proof.harnessProvenance.sourceStatus, "");
|
||||
assert.equal(proof.measurementSha256, first.candidate.measurementSha256, "H4d confirmation must bind the same measured harness bytes");
|
||||
assert.equal(proof.measurementSha256, first.candidate.measurementSha256, "H4e confirmation must bind the same measured harness bytes");
|
||||
assert.deepEqual(proof.resourceBudgets, ORIGINAL_BUDGETS);
|
||||
assert.equal(proof.complete, true); assert.equal(proof.qualificationOnly, false); assert.equal(proof.diagnosticOnly, false);
|
||||
assert.equal(proof.performanceAcceptanceEligible, true); assert.equal(proof.serverArtifact, "compiled");
|
||||
@@ -447,12 +450,15 @@ if (h5 && phase === "compare" && candidatePolicy === "final-b2") {
|
||||
const latestCandidateEnd = Math.max(...pairs.map(pair => Date.parse(pair.candidate.owningPlaywright.startedAt) + pair.candidate.owningPlaywright.durationMs));
|
||||
assert.ok(Date.parse(owner.startedAt) > latestCandidateEnd, "Original confirmation must follow all candidate arms");
|
||||
assert.ok(pairs.every(pair => proof.companyId !== pair.baseline.companyId && proof.companyId !== pair.candidate.companyId), "Original confirmation needs a fresh disposable company/database");
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4d-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
const inventory = await validateArmInventory(path.resolve(arg("original-h4e-inventory")!), { evidence: proof, owningPlaywright: owner });
|
||||
for (const field of ["manifestFileSha256", "aggregateSha256", "qualifiedReceiptSha256", "inventorySummarySha256", "revision"] as const)
|
||||
assert.equal(inventory[field], first.candidate.armInventory[field], "Original confirmation used a different full qualified artifact inventory");
|
||||
originalH4dConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision };
|
||||
originalH4eConfirmation = { required: true, passed: true, owner, inventory, sourceRevision: proof.sourceRevision,
|
||||
retryObservations: proof.cases.map((item: any) => ({ name: item.name,
|
||||
...assertRecordedFailures(item.recordedFailures, item, proof.companyId),
|
||||
transitions: item.terminalObservation.transitions, recordedFailures: item.recordedFailures })) };
|
||||
} catch (error) {
|
||||
originalH4dConfirmation.reason = error instanceof Error ? error.message : String(error);
|
||||
originalH4eConfirmation.reason = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -478,6 +484,34 @@ const measurements = names.flatMap(name => {
|
||||
samples };
|
||||
});
|
||||
});
|
||||
const retryObservations = names.map(name => {
|
||||
const samples = pairs.map(pair => Object.fromEntries((["baseline", "candidate"] as const).map(arm => {
|
||||
const item = pair[arm].cases.find((value: any) => value.name === name);
|
||||
const value = item.h5?.terminalObservation ?? item.terminalObservation;
|
||||
const recorded = assertRecordedFailures(item.recordedFailures, item, pair[arm].companyId);
|
||||
return [arm, { runs: item.runs, ...recorded,
|
||||
retryRate: recorded.uniqueEverFailedRuns / item.runs,
|
||||
errorClassRates: Object.fromEntries(Object.entries(recorded.recordedErrorClasses).map(([code, count]) => [code, Number(count) / item.runs])),
|
||||
transitionCount: value.transitions.length, transitions: value.transitions, recordedFailures: item.recordedFailures }];
|
||||
}))) as Array<Record<string, any>>;
|
||||
const observedErrorClasses = [...new Set(samples.flatMap(sample => ["baseline", "candidate"]
|
||||
.flatMap(arm => Object.keys(sample[arm].recordedErrorClasses))))].sort();
|
||||
const rates = [null, ...observedErrorClasses].map(errorClass => {
|
||||
const values = samples.map(sample => {
|
||||
const baselineCount = errorClass === null ? sample.baseline.uniqueEverFailedRuns : sample.baseline.recordedErrorClasses[errorClass] ?? 0;
|
||||
const candidateCount = errorClass === null ? sample.candidate.uniqueEverFailedRuns : sample.candidate.recordedErrorClasses[errorClass] ?? 0;
|
||||
assert.equal(sample.baseline.runs, sample.candidate.runs);
|
||||
const runs = sample.baseline.runs;
|
||||
return { runs, baselineCount, candidateCount, baseline: baselineCount / runs, candidate: candidateCount / runs,
|
||||
increaseCount: candidateCount - baselineCount, increase: (candidateCount - baselineCount) / runs };
|
||||
});
|
||||
return { errorClass, samples: values, medianIncrease: median(values.map(value => value.increase)),
|
||||
medianIncreaseCount: median(values.map(value => value.increaseCount)),
|
||||
aaNoiseEnvelopeRuns: phase === "aa" ? Math.max(...values.map(value => Math.abs(value.increaseCount))) : undefined,
|
||||
aaNoiseEnvelopeAbsolute: phase === "aa" ? Math.max(...values.map(value => Math.abs(value.increase))) : undefined };
|
||||
});
|
||||
return { name, pairCount: samples.length, observedErrorClasses, samples, rates };
|
||||
});
|
||||
const result: Record<string, any> = {
|
||||
scope: "Compiled-app streaming with100 independent normal agents and the original local Stop workload; real cloud Stop is separately required",
|
||||
agentTopology: first.baseline.agentTopology,
|
||||
@@ -490,11 +524,11 @@ const result: Record<string, any> = {
|
||||
gatesSha256: createHash("sha256").update(await readFile(new URL("./native-ingestion-performance-gates.mjs", import.meta.url))).digest("hex"),
|
||||
comparatorSha256: createHash("sha256").update(await readFile(new URL(import.meta.url))).digest("hex"),
|
||||
manifestSha256: createHash("sha256").update(await readFile(manifestPath)).digest("hex"), measurements,
|
||||
primaryLoadedWorkload: "burst512", complete: true,
|
||||
primaryLoadedWorkload: "burst512", complete: true, retryObservations,
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, candidatePolicy, h5Assessments,
|
||||
artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4dConfirmation,
|
||||
originalProtocolAcceptance: "Baseline SLA misses are retained; H5b complete observation is not an original-deadline PASS",
|
||||
finalH4dFunctionalPassRequired: candidatePolicy === "final-b2" } : {}),
|
||||
artifactInventories: { baseline: first.baseline.armInventory, candidate: first.candidate.armInventory }, originalH4eConfirmation,
|
||||
originalProtocolAcceptance: "Baseline SLA misses are retained; H5c complete observation is not an original-deadline PASS",
|
||||
finalH4eFunctionalPassRequired: candidatePolicy === "final-b2" } : {}),
|
||||
};
|
||||
if (phase === "compare") {
|
||||
assert.ok(arg("noise-report"), "Comparison requires its frozen A/A --noise-report");
|
||||
@@ -529,14 +563,33 @@ if (phase === "compare") {
|
||||
noiseExceedingPairedWins: paired.noiseExceedingPairedWins, requiredPairedWins: paired.requiredPairedWins,
|
||||
passed: improvementRequired ? improvement : noRegression };
|
||||
});
|
||||
assert.equal(noise.retryObservations?.length, names.length, "A/A must retain complete retry-rate calibration");
|
||||
result.retryGates = retryObservations.flatMap(item => {
|
||||
const control = noise.retryObservations.find((entry: any) => entry.name === item.name);
|
||||
assert.equal(control?.pairCount, pairs.length, "A/A retry calibration must retain every pair");
|
||||
return item.rates.map(rate => {
|
||||
const knownClass = rate.errorClass === null || control.observedErrorClasses.includes(rate.errorClass);
|
||||
const reference = control.rates.find((entry: any) => entry.errorClass === rate.errorClass);
|
||||
const envelope = reference?.aaNoiseEnvelopeAbsolute ?? 0;
|
||||
assert.ok(Number.isFinite(envelope) && envelope >= 0 && envelope <= 1, "Invalid frozen absolute retry-rate noise");
|
||||
const envelopeRuns = reference?.aaNoiseEnvelopeRuns ?? 0;
|
||||
assert.ok(Number.isSafeInteger(envelopeRuns) && envelopeRuns >= 0
|
||||
&& envelope === envelopeRuns / rate.samples[0].runs, "Retry-rate noise must match its exact run counts");
|
||||
const pairedPassed = rate.samples.every(sample => sample.increaseCount <= envelopeRuns);
|
||||
return { name: item.name, errorClass: rate.errorClass, knownClass, noiseEnvelopeAbsolute: envelope,
|
||||
medianIncrease: rate.medianIncrease, pairedIncreases: rate.samples.map(sample => sample.increase),
|
||||
passed: knownClass && pairedPassed && rate.medianIncreaseCount <= envelopeRuns };
|
||||
});
|
||||
});
|
||||
result.performancePassed = result.gates.every((entry: any) => entry.passed)
|
||||
&& result.retryGates.every((entry: any) => entry.passed)
|
||||
&& (!h5 || h5Assessments.every(entry => entry.candidateSlaPolicyPassed));
|
||||
result.acceptancePassed = result.performancePassed && (!originalH4dConfirmation || originalH4dConfirmation.passed);
|
||||
result.acceptancePassed = result.performancePassed && (!originalH4eConfirmation || originalH4eConfirmation.passed);
|
||||
}
|
||||
if (arg("output")) await writeFile(arg("output")!, JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (process.argv.includes("--require-performance")) {
|
||||
assert.equal(h5, true, "H5b performance-only gating is explicit; default H4d keeps original deadlines");
|
||||
assert.equal(h5, true, "H5c performance-only gating is explicit; default H4e keeps original deadlines");
|
||||
assert.equal(phase, "compare");
|
||||
assert.equal(result.performancePassed, true, "H5 performance/SLA gates failed; retain results and diagnose");
|
||||
}
|
||||
|
||||
@@ -4,4 +4,4 @@ console.log(JSON.stringify({ dryRun: true, executionAuthorized: false, workloadS
|
||||
registration: H5_REGISTRATION, requiredBeforeRelease: ['Root-reviewed exact driver/comparator hashes',
|
||||
'New functional observation qualification', 'Fresh B0 and B1 A/A before further product implementation',
|
||||
'Root-reviewed frozen A/A report hashes before candidate stages', 'Final B2 every-arm original SLA pass',
|
||||
'Later H4d default-mode full functional pass on the same source, geometry and final SUT'] }, null, 2));
|
||||
'Later H4e default-mode full functional pass on the same source, geometry and final SUT'] }, null, 2));
|
||||
@@ -1,12 +1,27 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { h5Enabled, H5_MODE, observeStreaming, originalTerminal, requireH5Candidate,
|
||||
H5_LIMITS, ORIGINAL_BUDGETS, prepareParticipantAgents } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
H5_LIMITS, ORIGINAL_BUDGETS, prepareParticipantAgents, observeRunSettlement } from '../tests/e2e/fixtures/native-ingestion-h5.mjs';
|
||||
|
||||
const timeout = milliseconds => Object.assign(new Error(`Timeout ${milliseconds}ms exceeded while waiting on the predicate`),
|
||||
{ matcherResult: { name: 'toBeGreaterThanOrEqual', pass: false } });
|
||||
|
||||
test('the actual paginated result counter counts accepted results in default H4e and observation H5c', async () => {
|
||||
const source = await readFile(new URL('../tests/e2e/native-event-ingestion.journey.ts', import.meta.url), 'utf8');
|
||||
const statements = source.split('\n').filter(line => line.includes('resultAcceptedEvents++'));
|
||||
assert.equal(statements.length, 1, 'Exercise the actual event-loop counter statement');
|
||||
const count = new Function('h5', 'rows', `let resultAcceptedEvents = 0;
|
||||
for (const row of rows) { const event = row.payload?.prpEvent; ${statements[0]} } return resultAcceptedEvents;`);
|
||||
for (const h5 of [false, true]) {
|
||||
assert.equal(count(h5, [{ eventType: 'run.result.accepted', payload: { prpEvent: { eventType: 'run.result.accepted' } } }]), 1);
|
||||
assert.equal(count(h5, [{ payload: { prpEvent: { eventType: 'run.result.accepted' } } }]), 1);
|
||||
assert.equal(count(h5, [{ eventType: 'run.result.proposed' }, { payload: { prpEvent: { eventType: 'item.delta' } } }]), 0);
|
||||
assert.equal(count(h5, []), 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('real frozen Playwright tiny poll exposes the matcher deadline shape used by H5', async () => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
assert.equal(require('@playwright/test/package.json').version, '1.62.1');
|
||||
@@ -39,10 +54,11 @@ test('real frozen Playwright can report its configured timeout before the nomina
|
||||
assessedAtMs, elapsedMs, observations, matcherName: failure.matcherResult.name, matcherPassed: failure.matcherResult.pass }));
|
||||
});
|
||||
|
||||
test('H5b requires its explicit mode and leaves default H4d selected', () => {
|
||||
test('H5c requires its explicit mode and leaves default H4e selected', () => {
|
||||
assert.equal(h5Enabled(undefined), false); assert.equal(h5Enabled(H5_MODE), true);
|
||||
assert.throws(() => h5Enabled('almost-h5'));
|
||||
assert.throws(() => h5Enabled('h5-observe-original-sla'), 'Old H5a observations use a different geometry');
|
||||
assert.throws(() => h5Enabled('h5b-observe-original-sla-independent-agents'), 'Old H5b observations stop before supported recovery');
|
||||
});
|
||||
|
||||
for (const fail of [false, true]) test(`participant setup uses at most8 workers, preserves index order and settles all attempts${fail ? ' after a failure' : ''}`, async () => {
|
||||
@@ -114,9 +130,96 @@ test('original terminal is not reached after120s miss; observational180s is dist
|
||||
assert.deepEqual(originalTerminal({ original: { status: 'failed' } }, terminal), {
|
||||
status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null });
|
||||
assert.equal(originalTerminal({ original: { status: 'passed' } }, terminal).status, 'passed');
|
||||
assert.equal(originalTerminal({ original: { status: 'passed' } }, { ...terminal, complete: false,
|
||||
censored: false, error: 'Terminal run failed' }).reason, 'original_terminal_failure');
|
||||
assert.equal(H5_LIMITS.terminalDrainageMs, ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
});
|
||||
|
||||
test('terminal observation keeps a supported native retry pending until same-run recovery within the original window', async () => {
|
||||
let clock = 1000, reads = 0; const calls = [], records = [];
|
||||
const states = [{ status: 'running', runtimeMode: 'native', nativePhase: 'executing' },
|
||||
{ status: 'failed', runtimeMode: 'native', nativePhase: 'retryable_failure', errorCode: 'provider_transport_failed' },
|
||||
{ status: 'succeeded', runtimeMode: 'native', nativePhase: 'committed' }];
|
||||
const value = await observeRunSettlement({ expectedRuns: 1, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
readRuns: async () => [{ id: 'same-run', status: states[Math.min(reads++, 2)].status }],
|
||||
readRun: async id => { calls.push(id); return { id, ...states[1] }; },
|
||||
poll: async (read, milliseconds) => {
|
||||
assert.equal(milliseconds, 180000);
|
||||
while (clock < 181000) { if (await read()) return; clock += 30000; }
|
||||
throw new Error('Unexpected terminal timeout');
|
||||
} });
|
||||
assert.equal(reads, 3); assert.deepEqual(calls, ['same-run']);
|
||||
assert.equal(value.runs[0].status, 'succeeded'); assert.equal(value.outcome.complete, true);
|
||||
assert.equal(value.outcome.startedAtMs, 1000); assert.equal(value.outcome.deadlineMs, 181000);
|
||||
assert.equal(value.outcome.assessedAtMs, 61000); assert.equal(value.outcome.censored, false);
|
||||
assert.equal(records.find(value => value.lastStatuses[0]?.nativePhase === 'retryable_failure').terminalRuns, 0);
|
||||
assert.deepEqual(value.outcome.transitions.map(row => row.status), ['running', 'failed', 'succeeded']);
|
||||
assert.equal(value.outcome.uniqueRetryableRuns, 1); assert.equal(value.outcome.recoveredRuns, 1);
|
||||
assert.equal(value.outcome.nonretryableFailures, 0);
|
||||
assert.deepEqual(value.outcome.retryableErrorClasses, { provider_transport_failed: 1 });
|
||||
});
|
||||
|
||||
test('a permanent native failure rejects immediately without waiting for another provider turn', async () => {
|
||||
let reads = 0; const records = [];
|
||||
await assert.rejects(observeRunSettlement({ expectedRuns: 2,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
readRuns: async () => { reads++; return [{ id: 'failed-run', status: 'failed' }, { id: 'still-running', status: 'running' }]; },
|
||||
readRun: async id => ({ id, status: 'failed', runtimeMode: 'native', nativePhase: 'terminal_failure' }),
|
||||
poll: async (read, milliseconds) => { assert.equal(milliseconds, 180000); await read(); } }), /Terminal run failed/);
|
||||
assert.equal(reads, 1); assert.equal(records.at(-1).complete, false); assert.equal(records.at(-1).censored, false);
|
||||
});
|
||||
|
||||
test('an unsettled retry is censored at the one unchanged180s deadline, never a completed latency', async () => {
|
||||
let clock = 1000, reads = 0; const records = [], calls = [];
|
||||
const error = Object.assign(new Error('Timeout 180000ms exceeded while waiting on the predicate'),
|
||||
{ matcherResult: { name: 'toBe', pass: false } });
|
||||
await assert.rejects(observeRunSettlement({ expectedRuns: 1, now: () => clock,
|
||||
onOutcome: value => records.push(structuredClone(value)),
|
||||
readRuns: async () => { reads++; return [{ id: 'same-run', status: 'failed' }]; },
|
||||
readRun: async id => ({ id, status: 'failed', runtimeMode: 'native', nativePhase: 'retryable_failure', errorCode: 'provider_transport_failed' }),
|
||||
poll: async (read, milliseconds) => {
|
||||
calls.push(milliseconds);
|
||||
while (clock < 181000) { if (await read()) return; clock += 30000; }
|
||||
throw error;
|
||||
} }), value => value === error);
|
||||
assert.deepEqual(calls, [180000]); assert.equal(reads, 6);
|
||||
const last = records.at(-1); assert.equal(last.deadlineMs, 181000); assert.equal(last.assessedAtMs, 181000);
|
||||
assert.equal(last.complete, false); assert.equal(last.censored, true); assert.equal(last.terminalRuns, 0);
|
||||
});
|
||||
|
||||
test('failed-list details may already show recovery; detail errors and unknown phases fail closed', async () => {
|
||||
const value = await observeRunSettlement({ expectedRuns: 1,
|
||||
readRuns: async () => [{ id: 'run', status: 'failed' }],
|
||||
readRun: async id => ({ id, status: 'succeeded', runtimeMode: 'native', nativePhase: 'committed' }),
|
||||
poll: async read => assert.equal(await read(), true) });
|
||||
assert.equal(value.runs[0].status, 'succeeded'); assert.equal(value.outcome.uniqueRetryableRuns, 0);
|
||||
const failure = new Error('normal detail GET failed');
|
||||
for (const readRun of [async () => { throw failure; },
|
||||
async id => ({ id, status: 'failed', runtimeMode: 'native', nativePhase: 'unknown' })]) {
|
||||
await assert.rejects(observeRunSettlement({ expectedRuns: 1,
|
||||
readRuns: async () => [{ id: 'run', status: 'failed' }], readRun, poll: async read => read() }),
|
||||
error => error === failure || error.message.includes('Terminal run failed'));
|
||||
}
|
||||
});
|
||||
|
||||
test('the real frozen Playwright terminal poll preserves detail errors and exposes its deadline matcher', async () => {
|
||||
const require = createRequire(import.meta.url), { expect } = require('@playwright/test');
|
||||
const failure = new Error('actual callback failure');
|
||||
await assert.rejects(expect.poll(async () => { throw failure; }, { timeout: 15 }).toBe(true), error => error === failure);
|
||||
await assert.rejects(expect.poll(async () => false, { timeout: 15, intervals: [5] }).toBe(true), error => {
|
||||
assert.equal(error.matcherResult?.name, 'toBe'); assert.equal(error.matcherResult.pass, false);
|
||||
assert.ok(error.message.includes('Timeout 15ms exceeded while waiting on the predicate')); return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('bounded state evidence fails on overflow instead of omitting a transition', async () => {
|
||||
let reads = 0;
|
||||
await assert.rejects(observeRunSettlement({ expectedRuns: 1,
|
||||
readRuns: async () => [{ id: 'run', status: 'running', errorCode: String(reads++) }], readRun: async () => assert.fail(),
|
||||
poll: async read => { for (let index = 0; index < 17; index++) await read(); } }), /evidence overflow/);
|
||||
});
|
||||
|
||||
test('cursor inherits only paired exact known misses; final candidate inherits none', () => {
|
||||
const base = { misses: ['burst512:streaming120'] };
|
||||
requireH5Candidate('cursor-nonregression', base, base);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS,
|
||||
import { H5_MODE, H5_PROTOCOL, H4E_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, H5_BASELINE_REVISIONS,
|
||||
originalTerminal, assessH5Arm, requireH5Candidate } from "../tests/e2e/fixtures/native-ingestion-h5.mjs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -77,6 +77,15 @@ const workloads = [
|
||||
{ name: "existing-history", runs: 10, events: 128, intervalMs: 0, prelude: 1000 },
|
||||
{ name: "operator-stop", runs: 1, events: 512, intervalMs: 100, prelude: 0, stop: true },
|
||||
];
|
||||
function syntheticSettlement(item) {
|
||||
const states = item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded",
|
||||
runtimeMode: null, nativePhase: null, errorCode: null }));
|
||||
return { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000,
|
||||
assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs,
|
||||
pendingRetryRuns: 0, lastStatuses: states, transitions: states.map(row => ({ observedAtMs: item.settledAtMs, ...row })),
|
||||
uniqueRetryableRuns: 0, recoveredRuns: 0, nonretryableFailures: 0, retryableErrorClasses: {}, complete: true, censored: false };
|
||||
}
|
||||
|
||||
function browserReport(revision, latency) {
|
||||
const companyId = "00000000-0000-4000-8000-000000000001", environmentId = "00000000-0000-4000-8000-000000000002";
|
||||
const localEnvironmentId = "00000000-0000-4000-8000-000000000003", controlAgentId = "00000000-0000-4000-8000-000000000004";
|
||||
@@ -84,7 +93,7 @@ function browserReport(revision, latency) {
|
||||
const remoteAgentIds = Array.from({ length: 100 }, (_, index) => id(9, 0, index));
|
||||
const transportPolicy = { streaming: "sandbox-remote", operatorStop: "local",
|
||||
realCloudStop: "Separate required qualification; fake-plugin void receipts grant no remote Stop authority" };
|
||||
return { protocolVersion: H4D_PROTOCOL, transport: "sandbox-remote-with-local-stop", transportPolicy, companyId,
|
||||
return { protocolVersion: H4E_PROTOCOL, transport: "sandbox-remote-with-local-stop", transportPolicy, companyId,
|
||||
bodyStartedAtMs: 1, agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: 100 },
|
||||
harnessProvenance: { sourceRevision: "b".repeat(40), sourceStatus: "", driverLockSha256: "8".repeat(64),
|
||||
sourceHashes: { "native-event-ingestion.journey.ts": "9".repeat(64), "fixtures/native-ingestion-h5.mjs": "a".repeat(64) } },
|
||||
@@ -143,7 +152,14 @@ function browserReport(revision, latency) {
|
||||
setupActivationConcurrency: 2, setupReadinessStartedAtMs: 100, readinessDeadlineMs: 300100, allReadyObservedAtMs: 700,
|
||||
participants: workload.runs, activations, activationWaves,
|
||||
epochMs: 1000, preludeProof: workload.prelude ? { individualDeltas: workload.prelude, uniqueSourceIdentities: workload.prelude, verifiedAtMs: 999 } : undefined,
|
||||
terminalObservation: syntheticSettlement({ ...workload, stop, runIds: rows.map(row => row.run_id), settledAtMs: 1000 + latency }),
|
||||
settledAtMs: 1000 + latency, stopRequestedAtMs: stop ? 1000 : undefined,
|
||||
recordedFailures: { kind: "recorded-run-failure-codes", transaction: "READ ONLY REPEATABLE READ",
|
||||
observedReadOnly: "on", observedIsolation: "repeatable read", queryStartedAtMs: 1002 + latency,
|
||||
checkedAtMs: 1003 + latency, connectionClosedAtMs: 1004 + latency,
|
||||
rows: rows.map(row => ({ run_id: row.run_id, company_id: companyId, agent_id: row.agent_id, runtime_mode: "native",
|
||||
status: stop ? "cancelled" : "succeeded", native_phase: "committed", current_error_code: stop ? "cancelled" : null, current_has_error: stop,
|
||||
recovered_type: null, recovered_schema: null, recovered_error_code: null, recovered_observed_at: null, finalization_attempt: 1 })) },
|
||||
stopAcknowledgedAtMs: stop ? 1000 + latency : undefined, remoteLeasesCheckedAtMs: stop ? undefined : 1001 + latency,
|
||||
runIds: rows.map(row => row.run_id), controllerAuthority: { ...snapshot, rows,
|
||||
...(stop ? { localJournal: { schema: "paperclip.runner.durable.control-plane-state.v1", sha256: "8".repeat(64), bytes: 512, checkedAtMs: 999,
|
||||
@@ -165,6 +181,7 @@ function browserReport(revision, latency) {
|
||||
intendedToVisible: { p95Ms: latency }, drainThroughTerminalMs: latency,
|
||||
stopThroughTerminalMs: latency, stopThroughAcknowledgementMs: latency,
|
||||
durable: rows.map((row, index) => ({ runId: row.run_id, token: String(index), ordered: true,
|
||||
resultAcceptedEvents: stop ? 0 : 1,
|
||||
uniqueSourceIdentities: stop ? 3 : workload.events, individualDeltas: stop ? 3 : workload.events, matchedSourceClocks: stop ? 3 : workload.events })),
|
||||
};
|
||||
}) };
|
||||
@@ -185,6 +202,8 @@ function h5Report(revision, loadedLatency) {
|
||||
item.firstVisibleAfterEmissionMs = latency; item.emissionToVisible = { medianMs: latency, p95Ms: latency };
|
||||
item.intendedToVisible = { p95Ms: latency };
|
||||
if (!item.stop) item.remoteLeasesCheckedAtMs = item.settledAtMs + 1;
|
||||
Object.assign(item.recordedFailures, { queryStartedAtMs: item.settledAtMs + 2,
|
||||
checkedAtMs: item.settledAtMs + 3, connectionClosedAtMs: item.settledAtMs + 4 });
|
||||
item.visible = Array.from({ length: item.stop ? 3 : item.events }, (_, index) => ({ index, visibleMs: item.settledAtMs }));
|
||||
const missed = latency > 120000, assessedAtMs = missed ? item.epochMs + 120000 : item.settledAtMs;
|
||||
const streaming = { original: { status: missed ? "failed" : "passed", reason: missed ? "original_streaming_deadline" : null,
|
||||
@@ -193,9 +212,7 @@ function h5Report(revision, loadedLatency) {
|
||||
lastObservedAtMs: missed ? assessedAtMs - 1 : assessedAtMs,
|
||||
error: missed ? "Timeout 120000ms exceeded while waiting on the predicate" : null },
|
||||
observation: { ceilingMs: 600000, deadlineMs: item.epochMs + 600000, complete: true, censored: false, completedAtMs: item.settledAtMs } };
|
||||
const terminalObservation = { startedAtMs: item.settledAtMs, deadlineMs: item.settledAtMs + 180000,
|
||||
assessedAtMs: item.settledAtMs, lastObservedAtMs: item.settledAtMs, observedRuns: item.runs, terminalRuns: item.runs,
|
||||
lastStatuses: item.runIds.map(runId => ({ runId, status: item.stop ? "cancelled" : "succeeded" })), complete: true, censored: false };
|
||||
const terminalObservation = syntheticSettlement(item);
|
||||
item.h5 = { streaming, terminalObservation, terminalOriginal: originalTerminal(streaming, terminalObservation),
|
||||
visibleCountByOriginalPaintDeadline: missed ? 0 : item.visible.length,
|
||||
independentDrain: { verifiedAtMs: item.settledAtMs + 10, elapsedFromEpochMs: latency + 10,
|
||||
@@ -221,7 +238,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "native-ingestion-h5-gates-"));
|
||||
const script = new URL("./compare-native-ingestion-browser.ts", import.meta.url);
|
||||
const save = async (name, value) => { const file = path.join(directory, name); await writeFile(file, JSON.stringify(value)); return file; };
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8" });
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, `--observation-mode=${H5_MODE}`, ...args], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
|
||||
const hash = value => createHash("sha256").update(value).digest("hex"), inventories = new Map();
|
||||
async function inventory(report, playwrightFile, name) {
|
||||
const variant = ({ [H5_BASELINE_REVISIONS.B0]: "B0", [H5_BASELINE_REVISIONS.B1]: "B1", ["c".repeat(40)]: "B2" })[report.sourceRevision];
|
||||
@@ -249,7 +266,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
sourceRevision: report.sourceRevision, compiledBuildRevision: report.compiledBuildRevision,
|
||||
playwrightReportSha256: hash(await readFile(playwrightFile)), ...inventories.get(variant) });
|
||||
}
|
||||
async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, duration = 500000 } = {}) {
|
||||
async function manifest(prefix, candidateLatency, { sameRevision = false, baselineRevision = "B0", candidateRevision = "B2", mutate, mutateArm, duration = 500000 } = {}) {
|
||||
const entries = [];
|
||||
for (let index = 0; index < 5; index++) {
|
||||
const entry = {};
|
||||
@@ -258,6 +275,7 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
const owner = { timeout: 1200000, expectedStatus: "passed", results: [{ status: "passed", retry: 0,
|
||||
duration, startTime: "2026-11-03T00:00:00.000Z", errors: [], attachments: [] }] };
|
||||
if (index === 0 && arm === "candidate") mutate?.(report, owner);
|
||||
mutateArm?.(report, { index, arm });
|
||||
owner.results[0].attachments.push({ name: "native-ingestion-evidence", contentType: "application/json",
|
||||
body: Buffer.from(JSON.stringify(report)).toString("base64") });
|
||||
entry[`${arm}File`] = await save(`${prefix}-${index}-${arm}.json`, report);
|
||||
@@ -270,16 +288,16 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
}
|
||||
try {
|
||||
const proof = JSON.parse(JSON.stringify(h5Report("B2", 1000)).replaceAll("00000000-", "10000000-"));
|
||||
proof.protocolVersion = H4D_PROTOCOL;
|
||||
proof.protocolVersion = H4E_PROTOCOL;
|
||||
proof.measurementSha256 = "h5-synthetic-fixed-fixture";
|
||||
// H4d defaults use the same frozen source/geometry as the observation arms.
|
||||
// H4e defaults use the same frozen source/geometry as the observation arms.
|
||||
for (const field of ["observationMode", "observationLimits", "observationComplete", "workloadSha256", "actualPlaywrightTimeoutMs"]) delete proof[field];
|
||||
for (const item of proof.cases) { delete item.h5; delete item.terminalStatuses; for (const row of item.durable) delete row.resultAcceptedEvents; }
|
||||
const originalProof = await save("original-h4d-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{
|
||||
for (const item of proof.cases) { item.terminalObservation = item.h5.terminalObservation; delete item.h5; }
|
||||
const originalProof = await save("original-h4e-playwright.json", { suites: [{ tests: [{ timeout: 900000, expectedStatus: "passed", results: [{
|
||||
status: "passed", retry: 0, duration: 5000, startTime: "2026-11-04T00:00:00.000Z", errors: [], attachments: [{
|
||||
name: "native-ingestion-evidence", body: Buffer.from(JSON.stringify(proof)).toString("base64") }] }] }] }] });
|
||||
const originalInventory = await inventory(proof, originalProof, "original-h4c");
|
||||
const finalProofArgs = [`--original-h4d-proof=${originalProof}`, `--original-h4d-inventory=${originalInventory}`];
|
||||
const finalProofArgs = [`--original-h4e-proof=${originalProof}`, `--original-h4e-inventory=${originalInventory}`];
|
||||
const noise = path.join(directory, "aa.json");
|
||||
const aa = run(["--phase=aa", `--pairs=${await manifest("aa", 181000, { sameRevision: true })}`, `--output=${noise}`]);
|
||||
assert.equal(aa.status, 0, aa.stderr);
|
||||
@@ -316,37 +334,96 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
assert.equal(pendingReport.performancePassed, true); assert.equal(pendingReport.acceptancePassed, false);
|
||||
const performanceOnly = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.equal(performanceOnly.status, 0, performanceOnly.stderr);
|
||||
const setRetries = (report, count, code = "provider_transport_failed", observed = true) => {
|
||||
const item = report.cases[3], value = item.h5.terminalObservation;
|
||||
value.transitions = [...item.runIds.slice(0, observed ? count : 0).map(runId => ({ observedAtMs: value.startedAtMs,
|
||||
runId, status: "failed", runtimeMode: "native", nativePhase: "retryable_failure", errorCode: code })),
|
||||
...value.lastStatuses.map(row => ({ observedAtMs: value.assessedAtMs, ...row }))];
|
||||
Object.assign(value, { uniqueRetryableRuns: observed ? count : 0, recoveredRuns: observed ? count : 0,
|
||||
retryableErrorClasses: observed && count ? { [code]: count } : {} });
|
||||
for (let index = 0; index < count; index++) Object.assign(item.recordedFailures.rows[index], {
|
||||
recovered_type: "object", recovered_schema: "paperclip.recovered_execution_failure.v1", recovered_error_code: code,
|
||||
recovered_observed_at: new Date(item.settledAtMs - 1).toISOString() });
|
||||
};
|
||||
const retryNoise = path.join(directory, "retry-aa.json");
|
||||
const retryAA = run(["--phase=aa", `--pairs=${await manifest("retry-aa", 181000, { sameRevision: true,
|
||||
mutateArm: (report, { arm }) => setRetries(report, arm === "candidate" ? 2 : 1) })}`, `--output=${retryNoise}`]);
|
||||
assert.equal(retryAA.status, 0, retryAA.stderr);
|
||||
const calibrated = JSON.parse(await readFile(retryNoise, "utf8")).retryObservations[3];
|
||||
assert.equal(calibrated.rates.find(rate => rate.errorClass === null).aaNoiseEnvelopeAbsolute, .01);
|
||||
assert.equal(calibrated.samples[0].baseline.uniqueRetryableRuns, 1);
|
||||
assert.equal(calibrated.samples[0].candidate.recoveredRuns, 2);
|
||||
const staleCurrent = run(["--candidate-policy=final-b2",
|
||||
`--pairs=${await manifest("stale-current-error", 1000, {
|
||||
mutateArm: report => setRetries(report, 1),
|
||||
mutate: report => Object.assign(report.cases[3].recordedFailures.rows[0], {
|
||||
current_has_error: true, current_error_code: "provider_transport_failed" }) })}`,
|
||||
`--noise-report=${retryNoise}`, "--require-performance"]);
|
||||
assert.notEqual(staleCurrent.status, 0, "An unresolved current error is not recovered merely because status says succeeded");
|
||||
assert.match(staleCurrent.stderr, /must clear their current execution error/);
|
||||
for (const [name, mutation, expectedPass] of [
|
||||
["retry-at-frozen-noise", (report, { arm }) => setRetries(report, arm === "candidate" ? 2 : 1), true],
|
||||
["one-arm-retry-storm", (report, { arm, index }) => setRetries(report, arm === "candidate" && index === 0 ? 3 : 1), false],
|
||||
["new-retry-class", (report, { arm }) => setRetries(report, 1, arm === "candidate" ? "new_transport_failure" : "provider_transport_failed"), false],
|
||||
]) {
|
||||
const output = path.join(directory, `${name}.json`);
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${await manifest(name, 1000, { mutateArm: mutation })}`,
|
||||
`--noise-report=${retryNoise}`, `--output=${output}`, "--require-performance"]);
|
||||
assert.equal(attempt.status === 0, expectedPass, attempt.stderr);
|
||||
const observed = JSON.parse(await readFile(output, "utf8"));
|
||||
assert.equal(observed.performancePassed, expectedPass);
|
||||
if (name === "one-arm-retry-storm") {
|
||||
const gate = observed.retryGates.find(gate => gate.name === "burst512" && gate.errorClass === null);
|
||||
assert.equal(gate.medianIncrease, 0); assert.equal(gate.pairedIncreases[0], .02); assert.equal(gate.passed, false);
|
||||
}
|
||||
if (name === "new-retry-class") assert.ok(observed.retryGates.some(gate => !gate.knownClass && !gate.passed));
|
||||
}
|
||||
const zeroBaselineRetries = run(["--candidate-policy=final-b2",
|
||||
`--pairs=${await manifest("zero-baseline-new-retry", 1000, { mutate: report => setRetries(report, 1) })}`,
|
||||
`--noise-report=${noise}`, "--require-performance"]);
|
||||
assert.notEqual(zeroBaselineRetries.status, 0, "A zero-retry A/A cannot silently admit a new retry class or rate");
|
||||
const earlyFile = path.join(directory, "early-recovery.json");
|
||||
const early = run(["--candidate-policy=final-b2",
|
||||
`--pairs=${await manifest("early-recovery", 1000, { mutate: report => setRetries(report, 1, "provider_transport_failed", false) })}`,
|
||||
`--noise-report=${noise}`, `--output=${earlyFile}`, "--require-performance"]);
|
||||
assert.notEqual(early.status, 0, "Recovery before the terminal observation window cannot vanish from failure rates");
|
||||
const earlyObservation = JSON.parse(await readFile(earlyFile, "utf8")).retryObservations[3].samples[0].candidate;
|
||||
assert.equal(earlyObservation.uniqueRetryableRuns, 0); assert.equal(earlyObservation.uniqueEverFailedRuns, 1);
|
||||
assert.equal(earlyObservation.allRecordedRecoveredRuns, 1);
|
||||
for (const [name, change, message] of [
|
||||
["h4c-too-early", (_value, owner) => { owner.results[0].startTime = "2026-11-02T00:00:00.000Z"; }, /follow all candidate/],
|
||||
["h4c-wrong-sut", value => { value.sourceRevision = H5_BASELINE_REVISIONS.B0; value.compiledBuildRevision = value.sourceRevision; }, /strictly equal/],
|
||||
["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /H4d default deadlines/],
|
||||
["h4c-observation-mode", value => { value.observationMode = H5_MODE; }, /H4e default deadlines/],
|
||||
["old-h4c-proof", value => { value.protocolVersion = "v4-remote-streaming-original-local-stop-clock-stream";
|
||||
value.harnessProvenance.sourceRevision = "7d7e7a88d562a95a1d407b76474763b666c181f5"; }, /Old H4c geometry/],
|
||||
["h4d-wrong-harness", value => { value.harnessProvenance.sourceRevision = "d".repeat(40); }, /same frozen harness source/],
|
||||
["h4d-wrong-measurement", value => { value.measurementSha256 = "old-h5a"; }, /same measured harness bytes/],
|
||||
["h4d-shared-agent", value => { value.cases[3].controllerAuthority.rows[1].agent_id = value.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/],
|
||||
["h4d-missing-stop-proof", value => { delete value.cases[5].localStopProof; }, /physical Stop/],
|
||||
["h4d-provider-still-alive", value => { value.cases[5].localStopProof.processes.providerAbsent = false; }, /provider remains alive/],
|
||||
["h4d-pending-local-lease", value => { value.cases[5].localStopProof.lease.status = "active"; }, /lease cleanup remains unsettled/],
|
||||
["h4d-wrong-local-lease", value => { value.cases[5].localStopProof.lease.id = value.cases[3].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4d-ack-timing-mismatch", value => { value.cases[5].stopAcknowledgedAtMs++; }, /acknowledgement timing boundary/],
|
||||
["h4d-missing-remote-leases", value => { delete value.cases[3].remoteLeases; }, /post-timing lease cleanup proof/],
|
||||
["h4d-pending-remote-lease", value => { value.cases[3].remoteLeases[0].cleanupStatus = "pending"; }, /lease cleanup remains unsettled/],
|
||||
["h4d-wrong-remote-lease", value => { value.cases[3].remoteLeases[0].id = value.cases[5].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4d-premature-remote-cleanup-proof", value => { value.cases[3].remoteLeasesCheckedAtMs = value.cases[3].settledAtMs - 1; }, /leases after recorded timing/],
|
||||
["h4d-wrong-authority-company", value => { value.cases[3].controllerAuthority.rows[0].company_id = "00000000-0000-4000-8000-000000099999"; }, /authority company mismatch/],
|
||||
["h4d-wrong-authority-run", value => { value.cases[3].controllerAuthority.rows[0].run_id = value.cases[5].runIds[0]; }, /authority run set mismatch/],
|
||||
["old-h4d-proof", value => { value.protocolVersion = "v4d-independent-agents-original-deadlines"; }, /Old H4c geometry/],
|
||||
["h4e-wrong-harness", value => { value.harnessProvenance.sourceRevision = "d".repeat(40); }, /same frozen harness source/],
|
||||
["h4e-wrong-measurement", value => { value.measurementSha256 = "old-h5a"; }, /same measured harness bytes/],
|
||||
["h4e-shared-agent", value => { value.cases[3].controllerAuthority.rows[1].agent_id = value.cases[3].controllerAuthority.rows[0].agent_id; }, /distinct normal agents/],
|
||||
["h4e-missing-stop-proof", value => { delete value.cases[5].localStopProof; }, /physical Stop/],
|
||||
["h4e-provider-still-alive", value => { value.cases[5].localStopProof.processes.providerAbsent = false; }, /provider remains alive/],
|
||||
["h4e-pending-local-lease", value => { value.cases[5].localStopProof.lease.status = "active"; }, /lease cleanup remains unsettled/],
|
||||
["h4e-wrong-local-lease", value => { value.cases[5].localStopProof.lease.id = value.cases[3].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4e-ack-timing-mismatch", value => { value.cases[5].stopAcknowledgedAtMs++; }, /acknowledgement timing boundary/],
|
||||
["h4e-missing-remote-leases", value => { delete value.cases[3].remoteLeases; }, /post-timing lease cleanup proof/],
|
||||
["h4e-pending-remote-lease", value => { value.cases[3].remoteLeases[0].cleanupStatus = "pending"; }, /lease cleanup remains unsettled/],
|
||||
["h4e-wrong-remote-lease", value => { value.cases[3].remoteLeases[0].id = value.cases[5].controllerAuthority.rows[0].lease_id; }, /strictly equal/],
|
||||
["h4e-premature-remote-cleanup-proof", value => { value.cases[3].remoteLeasesCheckedAtMs = value.cases[3].settledAtMs - 1; }, /leases after recorded timing/],
|
||||
["h4e-wrong-authority-company", value => { value.cases[3].controllerAuthority.rows[0].company_id = "00000000-0000-4000-8000-000000099999"; }, /authority company mismatch/],
|
||||
["h4e-wrong-authority-run", value => { value.cases[3].controllerAuthority.rows[0].run_id = value.cases[5].runIds[0]; }, /authority run set mismatch/],
|
||||
["h4e-missing-settlement", value => { delete value.cases[3].terminalObservation; }, /canonical run settlement/],
|
||||
["h4e-missing-recorded-failures", value => { delete value.cases[3].recordedFailures; }, /post-timing recorded failure proof/],
|
||||
]) {
|
||||
const report = JSON.parse(await readFile(originalProof, "utf8")), owner = report.suites[0].tests[0],
|
||||
value = JSON.parse(Buffer.from(owner.results[0].attachments[0].body, "base64").toString());
|
||||
change(value, owner); owner.results[0].attachments[0].body = Buffer.from(JSON.stringify(value)).toString("base64");
|
||||
const file = await save(`${name}-playwright.json`, report), sidecar = await inventory(value, file, name), output = path.join(directory, `${name}-result.json`);
|
||||
const attempt = run(["--candidate-policy=final-b2", `--pairs=${goodPairs}`, `--noise-report=${noise}`,
|
||||
`--original-h4d-proof=${file}`, `--original-h4d-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]);
|
||||
`--original-h4e-proof=${file}`, `--original-h4e-inventory=${sidecar}`, `--output=${output}`, "--require-acceptance"]);
|
||||
assert.notEqual(attempt.status, 0, name);
|
||||
const rejected = JSON.parse(await readFile(output, "utf8"));
|
||||
assert.equal(rejected.performancePassed, true); assert.equal(rejected.acceptancePassed, false);
|
||||
assert.match(rejected.originalH4dConfirmation.reason, message);
|
||||
assert.match(rejected.originalH4eConfirmation.reason, message);
|
||||
}
|
||||
const badOutput = path.join(directory, "still-misses.json");
|
||||
const bad = run(["--candidate-policy=final-b2", `--pairs=${await manifest("still-misses", 150000)}`,
|
||||
@@ -357,6 +434,17 @@ test("H5 CLI retains baseline SLA failure, rejects final SLA misses and requires
|
||||
assert.equal(rejected.h5Assessments[0].candidateSlaPolicyPassed, false);
|
||||
for (const [name, mutate, message] of [
|
||||
["old-h5a-protocol", report => { report.protocolVersion = "v5-complete-observation-original-sla"; }, /strictly equal/],
|
||||
["old-h5b-protocol", report => { report.protocolVersion = "v5b-independent-agents-complete-observation-original-sla"; }, /strictly equal/],
|
||||
["retry-ledger-hidden", report => { setRetries(report, 1); report.cases[3].h5.terminalObservation.uniqueRetryableRuns = 0; }, /Retry ledger/],
|
||||
["missing-state-history", report => { delete report.cases[3].h5.terminalObservation.transitions; }, /state history/],
|
||||
["missing-recorded-failures", report => { delete report.cases[3].recordedFailures; }, /post-timing recorded failure proof/],
|
||||
["wrong-recorded-run", report => { report.cases[3].recordedFailures.rows[0].run_id = report.cases[5].runIds[0]; }, /run set mismatch/],
|
||||
["wrong-recorded-company", report => { report.cases[3].recordedFailures.rows[0].company_id = "wrong"; }, /company mismatch/],
|
||||
["wrong-recovered-schema", report => { setRetries(report, 1); report.cases[3].recordedFailures.rows[0].recovered_schema = "unknown"; }, /strictly equal/],
|
||||
["missing-recorded-code", report => { setRetries(report, 1); report.cases[3].recordedFailures.rows[0].recovered_error_code = null; }, /closed recorded error code/],
|
||||
["premature-recorded-snapshot", report => { report.cases[3].recordedFailures.queryStartedAtMs = report.cases[3].settledAtMs - 1; }, /post-timing failure snapshot/],
|
||||
["unclosed-recorded-snapshot", report => { delete report.cases[3].recordedFailures.connectionClosedAtMs; }, /post-timing failure snapshot/],
|
||||
["stop-recorded-provider-failure", report => { report.cases[5].recordedFailures.rows[0].current_error_code = "provider_transport_failed"; }, /recorded failed run must have recovered/],
|
||||
["censored", report => { report.cases[3].h5.streaming.observation.censored = true; }, /false/],
|
||||
["terminal-failed", report => { report.cases[3].terminalStatuses[99].status = "failed"; }, /successful drainage/],
|
||||
["result-missing", report => { report.cases[3].durable[99].resultAcceptedEvents = 0; }, /persisted result/],
|
||||
@@ -402,7 +490,7 @@ test("browser CLI rejects the three-pairs-above-noise counterexample and ineligi
|
||||
}
|
||||
return save(`${prefix}-pairs.json`, entries);
|
||||
}
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, ...args], { encoding: "utf8" });
|
||||
const run = args => spawnSync(process.execPath, [script.pathname, ...args], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
|
||||
try {
|
||||
const noise = path.join(directory, "noise.json");
|
||||
const aa = run(["--phase=aa", `--pairs=${await manifest("aa", Array(5).fill(0.10), true)}`, `--output=${noise}`]);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Native ingestion experiment helpers
|
||||
|
||||
For fresh cursor results and the current interleaved source reproduction, see [the new source report](../../doc/benchmarks/2026-11-03-native-event-ingestion-interleaved.md). The frozen runner below preserves the earlier experiment. Current application performance requires the separately qualified H4c driver and fresh comparisons; its gates remain open.
|
||||
For fresh cursor results and the current interleaved source reproduction, see [the new source report](../../doc/benchmarks/2026-11-03-native-event-ingestion-interleaved.md). The frozen runner below preserves the earlier experiment. Current application performance requires the independently frozen [H5c driver and helpers](h5c/README.md), successful qualification and fresh comparisons; its gates remain open. The older H4c runner below remains historical.
|
||||
|
||||
These helpers reproduce the experiment registered on November 3, 2026. They
|
||||
retain its immutable application revisions, harness revision, workload hashes,
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# Frozen H5c application helpers
|
||||
|
||||
This package preserves the exact four reviewed helpers listed in `sources.json`.
|
||||
No embedded path or code was changed for packaging. The owning run receipt
|
||||
identifies the bytes actually executed; copying source into a delivery commit
|
||||
establishes no application or performance result.
|
||||
|
||||
The driver is the independent tag `native-ingestion-h5c`, annotated tag object
|
||||
`0ff37cd81dd922ce9def66fa656549e8e68d6a50`, commit
|
||||
`e431ec63d41fe90c6fd68bd2fc2bb969feeca917`. Its lock SHA256 is
|
||||
`54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8`.
|
||||
The delivery checkout has a different lock. Reproduce with that immutable driver,
|
||||
including later H4e confirmation, even though its seven changed source files are
|
||||
also included in this PR. The [frozen protocol](../../../tests/e2e/fixtures/native-ingestion-h5.md)
|
||||
contains the workload, settlement rules and pass/fail gates.
|
||||
|
||||
## Why H5c exists
|
||||
|
||||
The historical H5b FULL B0 attempt failed qualification. It retained 51,200
|
||||
lossless durable deltas and 100 accepted results across 100 independent agents,
|
||||
but its terminal snapshot showed 83 succeeded runs and 17 retryable failures.
|
||||
Those 17 lacked timely provider drain receipts. The observer stopped the app
|
||||
before their first supported retry, scheduled 30 seconds later. Existing-history
|
||||
and Stop were not reached. That failed attempt remains retained; neither later
|
||||
recovery nor a root cause for the late receipts was established.
|
||||
|
||||
H5c keeps failed native `retryable_failure` runs pending inside the same original
|
||||
180-second terminal window. It uses normal run-detail reads, records bounded
|
||||
state transitions, and reads the exact run set once after timing to retain current
|
||||
and recovered error classes. Recovery is performed only by the normal app. Final
|
||||
success, durable results, exact deltas and lease cleanup remain mandatory. Fresh
|
||||
A/A freezes the observed retry-rate envelope; candidate comparisons reject new
|
||||
recorded error classes and rate increases outside that envelope. Retained
|
||||
recovery metadata records the last failure, not an exhaustive failure history.
|
||||
|
||||
The original 120-second streaming, 180-second terminal and 900-second invocation
|
||||
limits remain unchanged. H5c can observe a known streaming miss for up to 600
|
||||
seconds and an invocation for 1200 seconds, while keeping that original miss a
|
||||
failure. Final B2 must pass every original SLA in every arm and later pass one
|
||||
H4e default-mode full invocation on the same source, geometry and qualified build.
|
||||
The 30-minute process watchdog bounds resources; it grants no performance pass.
|
||||
|
||||
## Reproduction layout and preparation
|
||||
|
||||
Use one quiet disposable Linux host with the recorded Node/pnpm, PostgreSQL,
|
||||
Chromium and Runner inputs. Normally install and build each exact app revision
|
||||
independently. Retain its build receipt and inventories for `server/dist`,
|
||||
`ui/dist`, `packages/db/src` and `packages/shared/src`. Keep the driver install,
|
||||
application dependencies and compiled output separate. Native target reuse is
|
||||
allowed only when all 15 native inputs match exactly; the historical three app
|
||||
builds reused a target with Rust tree `bbe29bba574ce8d5cfddf2f7461ae0eb5983df07`.
|
||||
|
||||
The recorded app directories and full revisions appear in `sources.json`: B0
|
||||
`bac53ec3`, B1 `07061a60`, B2 `ca7cba17`. Qualified build receipt SHA256 is
|
||||
`a1259ba760f3cbbe44817e333f9f620bcf3769fb0a6c16f9ce2f0f5b2ed60d94`.
|
||||
Those bytes describe the recorded builds; a source checkout alone is no compiled
|
||||
attestation. The old B2 is the unoptimized batching base. A final Rust candidate
|
||||
requires a new immutable source, normal build and qualified inventory. The
|
||||
orchestrator rejects the old B2 as a final candidate.
|
||||
|
||||
Keep the recorded layout under `/home/ubuntu/native-ingestion`: app trees in
|
||||
`aligned-baseline`, `aligned-cursors`, `aligned-batching`; driver in `harness-h5c`;
|
||||
build receipts and inventory summaries at their recorded locations; helpers in
|
||||
`helpers` under their original basenames. Verify every copied helper against
|
||||
`sources.json`. Path-portable derivatives need separate hashes and provenance.
|
||||
Run this setup from the tools checkout; the driver path must not exist yet.
|
||||
|
||||
```sh
|
||||
TOOLS_CHECKOUT="$(git rev-parse --show-toplevel)"
|
||||
DRIVER_ROOT=/home/ubuntu/native-ingestion/harness-h5c
|
||||
HELPERS_ROOT=/home/ubuntu/native-ingestion/helpers
|
||||
git -C "$TOOLS_CHECKOUT" show-ref --verify --quiet refs/tags/native-ingestion-h5c ||
|
||||
git -C "$TOOLS_CHECKOUT" fetch origin tag native-ingestion-h5c
|
||||
test "$(git -C "$TOOLS_CHECKOUT" rev-parse 'native-ingestion-h5c^{commit}')" = e431ec63d41fe90c6fd68bd2fc2bb969feeca917
|
||||
git -C "$TOOLS_CHECKOUT" worktree add --detach "$DRIVER_ROOT" native-ingestion-h5c
|
||||
pnpm --dir "$DRIVER_ROOT" install --frozen-lockfile --offline --ignore-scripts
|
||||
mkdir -p "$HELPERS_ROOT"
|
||||
for helper in native-ingestion-run-h5c-browser.mjs native-ingestion-linux-h5c-qualification.py native-ingestion-h5c-linux-readiness.mjs native-ingestion-validate-h5c-evidence.mjs; do
|
||||
cp "$TOOLS_CHECKOUT/scripts/native-ingestion/h5c/$helper" "$HELPERS_ROOT/$helper"
|
||||
done
|
||||
node "$DRIVER_ROOT/scripts/describe-native-ingestion-h5.mjs"
|
||||
node "$HELPERS_ROOT/native-ingestion-run-h5c-browser.mjs"
|
||||
```
|
||||
|
||||
The last two commands are dry runs. They launch no app or performance arm. The
|
||||
readiness helper checks actual B0/B1 file inventories, native target, PATH,
|
||||
preferred-UI-shadow absence and physical quiet. Its output directory
|
||||
`evidence/h5c/setup` must exist and its receipt files must be new. An output-path
|
||||
change for another readiness check must have its own retained helper hash.
|
||||
|
||||
## Qualification and comparisons
|
||||
|
||||
Run one released operation at a time, with no competing builds, tests, profiles
|
||||
or benchmarks. Preserve every failed or censored attempt. After actual build and
|
||||
quiet checks, qualify SMALL first with a new output name:
|
||||
|
||||
```sh
|
||||
python3 "$HELPERS_ROOT/native-ingestion-linux-h5c-qualification.py" --tree=B0 --name=NEW-SMALL
|
||||
node "$HELPERS_ROOT/native-ingestion-validate-h5c-evidence.mjs" /home/ubuntu/native-ingestion/evidence/h5c/qualification/NEW-SMALL "$DRIVER_ROOT"
|
||||
```
|
||||
|
||||
Inspect its actual raw evidence and cleanup before separately releasing FULL:
|
||||
|
||||
```sh
|
||||
python3 "$HELPERS_ROOT/native-ingestion-linux-h5c-qualification.py" --tree=B0 --full --name=NEW-FULL
|
||||
node "$HELPERS_ROOT/native-ingestion-validate-h5c-evidence.mjs" /home/ubuntu/native-ingestion/evidence/h5c/qualification/NEW-FULL "$DRIVER_ROOT"
|
||||
```
|
||||
|
||||
The post-run validator binds exact driver helpers, SUT revision, FULL/SMALL case
|
||||
matrix, the owning Playwright attempt and its attached evidence, settlement,
|
||||
recorded errors and external cleanup. It creates a new qualification-only
|
||||
summary. It cannot turn qualification into an accepted A/A sample.
|
||||
|
||||
The browser orchestrator prints its five stages without arguments. Execution
|
||||
requires explicit byte-frozen `--release=...`, `--stage=...`, `--output-root=...`
|
||||
and `--execute`. `--validate-release` checks bindings without starting an arm.
|
||||
After successful FULL qualification, forecast the full experiment duration and
|
||||
freeze fresh B0 A/A and B1 A/A before any further product optimization. Each stage
|
||||
has five alternating pairs; all 50 arms are fresh. Loaded browser latency and
|
||||
final backlog drainage must improve beyond frozen noise in the median and at
|
||||
least four of five pairs; sparse streaming and control must not regress. The
|
||||
later H4e proof finalizes the same sealed comparison samples with
|
||||
`--original-h4e-proof`, `--original-h4e-inventory` and `--require-acceptance`.
|
||||
|
||||
At packaging, only one H5c SMALL B0 qualification had been released. No H5c
|
||||
runtime pass, A/A suitability or browser gain was established. Historical H5b
|
||||
helpers remain in [h5b](../h5b/README.md). Run receipts and the dated evidence
|
||||
report establish later results; this source package claims none.
|
||||
@@ -0,0 +1,29 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {readFile,writeFile,realpath} from 'node:fs/promises';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
import {createHash} from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
const base='/home/ubuntu/native-ingestion',helper=base+'/helpers/native-ingestion-run-h5c-browser.mjs',out=base+'/evidence/h5c/setup';
|
||||
const sha=b=>createHash('sha256').update(b).digest('hex');
|
||||
assert.equal(sha(await readFile(helper)),'6b6376b01de6cdc15ca05343a0bdd5c00424f1d6c546b9b4d61cd76716b253c4');
|
||||
const h=await import(helper);
|
||||
const quiet=await h.physicalQuietGuard({receiptFile:out+'/linux-quiet-readiness.json',phase:'pre-arm',armStage:'H5c-SETUP-ONLY'});
|
||||
const receiptFile=base+'/aligned-4a83-builds-qualified.json',summaryFile=base+'/evidence/aligned-4a83/environment/artifact-inventory-summary.json';
|
||||
const receipt=JSON.parse(await readFile(receiptFile,'utf8')),summary=JSON.parse(await readFile(summaryFile,'utf8'));
|
||||
const results=[];
|
||||
for(const [variant,directory] of [['B0','aligned-baseline'],['B1','aligned-cursors']]) {
|
||||
const row=receipt.results.find(v=>v.variant===variant),i=summary.inventories[variant],native=receipt.nativeStageHashes.find(v=>v.variant===variant);
|
||||
const descriptor={appRoot:base+'/'+directory,revision:row.revision,qualifiedReceiptFile:receiptFile,qualifiedReceiptSha256:sha(await readFile(receiptFile)),inventorySummaryFile:summaryFile,inventorySummarySha256:sha(await readFile(summaryFile)),manifestFile:path.join(path.dirname(summaryFile),i.manifest),manifestFileSha256:i.manifestSha256,aggregateSha256:i.aggregateSha256,binarySha256:native.binarySha256,rustSourceTree:receipt.rustSourceTree};
|
||||
const q=await h.validateVariantBuild(variant,descriptor),live=await h.validateActualInventory(descriptor,q.manifest),ui=await h.assertNoPreferredUiShadow(descriptor.appRoot);
|
||||
assert.equal(execFileSync('git',['rev-parse','HEAD'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),descriptor.revision);
|
||||
assert.equal(execFileSync('git',['status','--short'],{cwd:descriptor.appRoot,encoding:'utf8'}).trim(),'');
|
||||
assert.equal(JSON.parse(await readFile(path.join(descriptor.appRoot,'server/dist/build-info.json'),'utf8')).commit,descriptor.revision);
|
||||
const target=path.join(descriptor.appRoot,'packages/paperclip-runner/runner/target'),binary=path.join(target,'release/paperclip-runnerd');
|
||||
assert.equal(await realpath(target),native.targetRealPath);assert.equal(sha(await readFile(binary)),native.binarySha256);
|
||||
results.push({variant,descriptor,live,ui,nativeTargetRealPath:await realpath(target)});
|
||||
}
|
||||
const pathNode=execFileSync('node',['-p','process.execPath'],{encoding:'utf8'}).trim();
|
||||
assert.equal(await realpath(pathNode),await realpath(process.execPath));
|
||||
const data={setupOnly:true,noSutBrowserProviderOrEvents:true,performanceAcceptanceEligible:false,atMs:Date.now(),helperSha256:sha(await readFile(helper)),harnessRevision:'e431ec63d41fe90c6fd68bd2fc2bb969feeca917',quiet,path:{PATH:process.env.PATH,nodeExecutable:process.execPath,pathResolvedNode:pathNode,nodeVersion:process.version},results};
|
||||
await writeFile(out+'/linux-readiness.json',JSON.stringify(data,null,2)+'\n',{flag:'wx'});
|
||||
console.log(JSON.stringify({complete:true,setupOnly:true,noSutBrowserProviderOrEvents:true,variants:results.map(v=>({variant:v.variant,fileCount:v.live.fileCount,aggregate:v.live.aggregateSha256,uiShadowAbsent:v.ui.preferredIndexAbsent})),quiet:quiet.passed,nodeVersion:process.version}));
|
||||
@@ -0,0 +1,58 @@
|
||||
import pathlib,subprocess,json,os,socket,datetime,hashlib,traceback,argparse
|
||||
p=argparse.ArgumentParser();p.add_argument('--full',action='store_true');p.add_argument('--tree',choices=['B0','B1','B2'],default='B2');p.add_argument('--name',required=True);a=p.parse_args()
|
||||
base=pathlib.Path('/home/ubuntu/native-ingestion');repo=base/'paperclip';h=base/'harness-h5c';out=base/'evidence/h5c/qualification'/a.name;out.mkdir(parents=True,exist_ok=False)
|
||||
pins={'B0':('aligned-baseline','bac53ec3bfeaf6546e7aaeaee516487b30534654'),'B1':('aligned-cursors','07061a601cdc20603c8c7a46f9ddbad7ac5f5939'),'B2':('aligned-batching','ca7cba171cd5198ed569d75012edc858041cbc44')};app=base/pins[a.tree][0]
|
||||
ref='e431ec63d41fe90c6fd68bd2fc2bb969feeca917';protocol='2265aabe9c7344f5d4b9c8b08a5c6bbd5571a6e2de9ae7b72a8d1733250e609f';comparator='9e01acaae40251cb9730aa8a7e548d451b986112c20eae7ffc31759cf221393c';lock='54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8'
|
||||
r={'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'qualificationOnly':True,'performanceAcceptanceEligible':False,'fullFixture':a.full,'sourceRevision':pins[a.tree][1],'harnessRevision':ref,'steps':[]}
|
||||
def now():return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
def save():(out/'owner-receipt.json').write_text(json.dumps(r,indent=2)+'\n')
|
||||
def cmd(name,args,cwd,env=None,timeout=None):
|
||||
row={'name':name,'argv':list(map(str,args)),'cwd':str(cwd),'startedAt':now()};r['steps'].append(row);save()
|
||||
with(out/(name+'.log')).open('xb')as log:result=subprocess.run(args,cwd=cwd,env=env,stdout=log,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
row.update({'exit':result.returncode,'finishedAt':now()});save();assert result.returncode==0,(name,result.returncode)
|
||||
try:
|
||||
assert(base/'aligned-4a83-builds.exit').read_text().strip()=='0'
|
||||
qualified=json.loads((base/'aligned-4a83-builds-qualified.json').read_text())
|
||||
assert {row['variant']:row['revision'] for row in qualified['results']}=={v:pin for v,(directory,pin) in pins.items()}
|
||||
rows=[]
|
||||
for q in pathlib.Path('/proc').iterdir():
|
||||
if not q.name.isdigit():continue
|
||||
try:
|
||||
exe=os.readlink(q/'exe')
|
||||
if pathlib.Path(exe).name in['node','postgres','paperclip-runner','rustc','cargo','chrome','chromium','chrome-headless-shell']:rows.append({'pid':int(q.name),'executable':exe})
|
||||
except OSError:pass
|
||||
assert not rows,rows
|
||||
for directory,pin in pins.values():
|
||||
root=base/directory;assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=root,text=True).strip()==pin
|
||||
assert not subprocess.check_output(['git','status','--short'],cwd=root,text=True).strip()
|
||||
assert json.loads((root/'server/dist/build-info.json').read_text())['commit']==pin
|
||||
assert h.exists(), 'Stage the exact independent driver before qualification'
|
||||
assert not(h/'node_modules').is_symlink();assert subprocess.check_output(['git','rev-parse','HEAD'],cwd=h,text=True).strip()==ref
|
||||
assert not subprocess.check_output(['git','status','--short'],cwd=h,text=True).strip()
|
||||
files=['native-event-ingestion.journey.ts','playwright-native-ingestion.config.ts','fixtures/in-feed-codex.mjs','fixtures/native-ingestion-bin/codex','fixtures/composer-stop-codex-loader.mjs','fixtures/native-ingestion-bootstrap.mjs','fixtures/native-ingestion-remote-server.mjs','fixtures/native-ingestion-protocol-qualification.mjs','fixtures/native-ingestion-h5.mjs']
|
||||
assert hashlib.sha256(b''.join((h/'tests/e2e'/f).read_bytes()for f in files)).hexdigest()==protocol
|
||||
assert hashlib.sha256((h/'scripts/compare-native-ingestion-browser.ts').read_bytes()).hexdigest()==comparator
|
||||
assert hashlib.sha256((h/'pnpm-lock.yaml').read_bytes()).hexdigest()==lock
|
||||
assert hashlib.sha256((h/'tests/e2e/fixtures/native-ingestion-h5.md').read_bytes()).hexdigest()=='5c78466770ecc3664339015d382ad6941109082976910a0f8ca2b88f78638c1f'
|
||||
r['resourceWatchdogMillis']=1800000;r['resourceWatchdogScope']='Resource-only including startup/test/teardown; original SLA and H5 observation unchanged';r['protocolSha256']=protocol;r['comparatorSha256']=comparator;r['driverLockSha256']=lock
|
||||
with socket.socket()as s:s.bind(('127.0.0.1',0));port=s.getsockname()[1]
|
||||
env=dict(os.environ)
|
||||
for key in ['DATABASE_URL','PAPERCLIP_RUNNER_STATE_DIR','PAPERCLIP_INGESTION_SERVER_COMMAND','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS','PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR','PAPERCLIP_INGESTION_BROWSER_CAPTURE','PAPERCLIP_INGESTION_SMOKE','NODE_OPTIONS','PAPERCLIP_INGESTION_OBSERVATION_MODE']:env.pop(key,None)
|
||||
binary=str(app/'packages/paperclip-runner/runner/target/release/paperclip-runnerd')
|
||||
overrides={'PAPERCLIP_INGESTION_OBSERVATION_MODE':'h5c-observe-original-sla-independent-agents-settlement','PAPERCLIP_INGESTION_TRANSPORT':'sandbox-remote-with-local-stop','PAPERCLIP_INGESTION_APP_ROOT':str(app),'PAPERCLIP_INGESTION_EVIDENCE_DIR':str(out),'PAPERCLIP_E2E_PORT':str(port),'PAPERCLIP_RUNNER_BINARY':binary,'PAPERCLIP_RUNNER_REMOTE_BINARY_PATH':binary,'PAPERCLIP_INGESTION_VARIANT':a.name+'-FUNCTIONAL-QUALIFICATION-ONLY','PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES':'Independent H5c frozen offline ignore-scripts driver install; application owns independent normal locked install/build'}
|
||||
if not a.full:overrides['PAPERCLIP_INGESTION_SMOKE']='1'
|
||||
env.update(overrides);r['environmentOverrides']=overrides;save();subprocess.run(['sync'],check=True)
|
||||
cmd('playwright',['timeout','--signal=INT','--kill-after=40s','30m','node','node_modules/@playwright/test/cli.js','test','--config','tests/e2e/playwright-native-ingestion.config.ts'],h,env,1845)
|
||||
r['playwrightPassed']=True;r['finishedAt']=now();save()
|
||||
except BaseException as error:r.update({'passed':False,'error':str(error),'traceback':traceback.format_exc(),'finishedAt':now()});save();(out/'owner.exit').write_text('1\n');raise
|
||||
|
||||
finally:
|
||||
helper=base/'helpers/native-ingestion-run-h5c-browser.mjs'
|
||||
guard="import net from 'node:net'; import {physicalQuietGuard} from '"+str(helper)+"'; const s=net.createServer(); await new Promise((ok,bad)=>{s.once('error',bad);s.listen(Number(process.argv[2]),'127.0.0.1',ok)}); await new Promise(ok=>s.close(ok)); await physicalQuietGuard({receiptFile:process.argv[1],phase:'post-arm',armStage:'H5c-QUALIFICATION-ONLY'});"
|
||||
try:
|
||||
result=subprocess.run(['node','--input-type=module','-e',guard,str(out/'physical-quiet-after.json'),str(r['environmentOverrides']['PAPERCLIP_E2E_PORT'])],check=True,capture_output=True,text=True)
|
||||
(out/'port-after.json').write_text(json.dumps({'complete':True,'port':r['environmentOverrides']['PAPERCLIP_E2E_PORT'],'checkedAt':now(),'nodeBindAndListenPassed':True})+'\n')
|
||||
if r.get('playwrightPassed'):
|
||||
r['passed']=True;r['cleanupComplete']=True;save();(out/'owner.exit').write_text('0\n')
|
||||
except BaseException as cleanup_error:
|
||||
r['externalQuietFailure']=str(cleanup_error);r['passed']=False;r['cleanupComplete']=False;save();(out/'owner.exit').write_text('1\n');raise
|
||||
@@ -0,0 +1,652 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFile, writeFile, mkdir, open, access, readdir, realpath, stat, lstat } from 'node:fs/promises';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import net from 'node:net';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
// Prospective H5 application orchestrator. No launch without --execute and a
|
||||
// byte-frozen root release. Original ce33 helper SHA is provenance only:
|
||||
// 2244e238642caafd488a7c86fb463376c8459ee0f06f8054555407a174698fa1.
|
||||
export const MODE = 'h5c-observe-original-sla-independent-agents-settlement';
|
||||
export const PROTOCOL = 'v5c-independent-agents-complete-settlement-original-sla';
|
||||
export const WORKLOAD_SHA256 = '3eea2f24c0c0c38134492a0a17e1b9a398335c085dd248f03b14c14bb5a19835';
|
||||
const H5_REVISION = 'e431ec63d41fe90c6fd68bd2fc2bb969feeca917';
|
||||
const OLD_NATIVE_TARGET = '/home/ubuntu/native-ingestion/aligned-baseline/packages/paperclip-runner/runner/target';
|
||||
const H5_PROTOCOL_SHA256 = '2265aabe9c7344f5d4b9c8b08a5c6bbd5571a6e2de9ae7b72a8d1733250e609f';
|
||||
const H5_COMPARATOR_SHA256 = '9e01acaae40251cb9730aa8a7e548d451b986112c20eae7ffc31759cf221393c';
|
||||
const H5_INVENTORY_HELPER_SHA256 = '76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9';
|
||||
const H5_PROTOCOL_DOCUMENT_SHA256 = '5c78466770ecc3664339015d382ad6941109082976910a0f8ca2b88f78638c1f';
|
||||
const INVENTORY_ROOTS = ['server/dist', 'ui/dist', 'packages/db/src', 'packages/shared/src'];
|
||||
export const RESOURCE_WATCHDOG_MILLIS = 30 * 60_000;
|
||||
export const STAGES = Object.freeze(['browser-aa-b0', 'browser-aa-b1', 'browser-b0-b1', 'browser-b1-b2', 'browser-b0-b2']);
|
||||
export const FINALIZE_STAGE = 'browser-finalize-h4e';
|
||||
const protocolFiles = ['native-event-ingestion.journey.ts', 'playwright-native-ingestion.config.ts', 'fixtures/in-feed-codex.mjs',
|
||||
'fixtures/native-ingestion-bin/codex', 'fixtures/composer-stop-codex-loader.mjs', 'fixtures/native-ingestion-bootstrap.mjs',
|
||||
'fixtures/native-ingestion-remote-server.mjs', 'fixtures/native-ingestion-protocol-qualification.mjs', 'fixtures/native-ingestion-h5.mjs'];
|
||||
const BASELINES = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aaeaee516487b30534654', B1: '07061a601cdc20603c8c7a46f9ddbad7ac5f5939' });
|
||||
const OLD_B2 = 'ca7cba171cd5198ed569d75012edc858041cbc44';
|
||||
const OLD_RECEIPT = 'a1259ba760f3cbbe44817e333f9f620bcf3769fb0a6c16f9ce2f0f5b2ed60d94';
|
||||
const OLD_SUMMARY = 'd3766fc4206d569d10fd02224433f1e9a2caa6992a7c62216e80af368aa4e44d';
|
||||
const OLD_BINARY = 'abb65bdc073991e428bd37f9809603ee7564543cca0ed0305dc231137c93070f';
|
||||
const OLD_RUST = 'bbe29bba574ce8d5cfddf2f7461ae0eb5983df07';
|
||||
const OLD_INVENTORIES = Object.freeze({
|
||||
B0: { manifestFileSha256: 'a1c797a75dd3174f414fc5529acf56783e1a3dfe7b22f1e460a87b60f6bff01d', aggregateSha256: 'a50720be2ce53f9222187d63609c5936f94b6a00c60df2ca112261abd1bf921e' },
|
||||
B1: { manifestFileSha256: 'df48e5c511a2f31e31180e05f18e2322ca4f384ea06f6f96a57de24d39849ef8', aggregateSha256: 'bca5b6fad62e70631c135a9ff377efeb587aba17ef0842c8fd82804ed99a81ab' },
|
||||
});
|
||||
const LOCK_SHA256 = '54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8';
|
||||
const GATES_SHA256 = 'ded8bcbb8f05732348f6520b9c5e41b7c35bc098997eb263a9dcb174eede75fc';
|
||||
const selfFile = fileURLToPath(import.meta.url);
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const getJson = async file => JSON.parse(await readFile(file, 'utf8'));
|
||||
const saveJson = async (file, value, fresh = false) => writeFile(file, JSON.stringify(value, null, 2) + '\n', fresh ? { flag: 'wx' } : undefined);
|
||||
const digest = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{64}$/.test(value), `Missing exact ${name} SHA-256`);
|
||||
const revision = (value, name) => assert.ok(typeof value === 'string' && /^[a-f0-9]{40}$/.test(value), `Missing immutable full ${name} revision`);
|
||||
const absolute = (value, name) => assert.ok(typeof value === 'string' && path.isAbsolute(value), `Missing absolute ${name} path`);
|
||||
const option = name => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3);
|
||||
|
||||
export function stagePlan(stage) {
|
||||
assert.ok(STAGES.includes(stage) || stage === FINALIZE_STAGE, 'Select one registered H5 stage');
|
||||
if (stage === FINALIZE_STAGE) return { stage, requiredVariants: ['B0', 'B1', 'B2'], priorStages: [...STAGES], fullArms: 0, finalizationOnly: true };
|
||||
const [baseline, candidate] = stage === STAGES[0] ? ['B0', 'B0'] : stage === STAGES[1] ? ['B1', 'B1']
|
||||
: stage === STAGES[2] ? ['B0', 'B1'] : stage === STAGES[3] ? ['B1', 'B2'] : ['B0', 'B2'];
|
||||
const aa = baseline === candidate;
|
||||
return { stage, baseline, candidate, aa, expectation: stage === STAGES[2] ? 'nonregression' : 'improvement',
|
||||
candidatePolicy: aa ? null : stage === STAGES[2] ? 'cursor-nonregression' : 'final-b2',
|
||||
requiredVariants: [...new Set([baseline, candidate])], priorStages: STAGES.slice(0, STAGES.indexOf(stage)),
|
||||
pairs: 5, fullArms: 10, noiseStage: aa ? null : `browser-aa-${baseline.toLowerCase()}` };
|
||||
}
|
||||
|
||||
async function hashedJson(file, expected, name, compressed = false) {
|
||||
absolute(file, name); digest(expected, name);
|
||||
const bytes = await readFile(file); assert.equal(hash(bytes), expected, `${name} bytes changed`);
|
||||
return { bytes, value: JSON.parse((compressed ? gunzipSync(bytes, { maxOutputLength: 128 * 1024 * 1024 }) : bytes).toString('utf8')) };
|
||||
}
|
||||
|
||||
export async function validateVariantBuild(variant, descriptor) {
|
||||
assert.ok(descriptor && typeof descriptor === 'object', `Required ${variant} qualified build is missing`);
|
||||
absolute(descriptor.appRoot, `${variant} appRoot`); revision(descriptor.revision, variant);
|
||||
digest(descriptor.binarySha256, `${variant} binary`); revision(descriptor.rustSourceTree, `${variant} Rust source tree`);
|
||||
const receipt = (await hashedJson(descriptor.qualifiedReceiptFile, descriptor.qualifiedReceiptSha256, `${variant} qualified receipt`)).value;
|
||||
const summary = (await hashedJson(descriptor.inventorySummaryFile, descriptor.inventorySummarySha256, `${variant} inventory summary`)).value;
|
||||
const manifest = (await hashedJson(descriptor.manifestFile, descriptor.manifestFileSha256, `${variant} inventory manifest`, true)).value;
|
||||
digest(descriptor.aggregateSha256, `${variant} aggregate inventory`);
|
||||
const rows = receipt.results?.filter(row => row.variant === variant);
|
||||
assert.equal(rows?.length, 1, `${variant} qualified receipt must select this variant exactly once`);
|
||||
assert.equal(rows[0].revision, descriptor.revision, `${variant} qualified receipt revision mismatch`);
|
||||
assert.equal(rows[0].compiledBuildRevision, descriptor.revision, `${variant} qualified compiled revision mismatch`);
|
||||
assert.equal(rows[0].sourceClean, true, `${variant} qualified source was not clean`);
|
||||
const native = receipt.nativeStageHashes?.filter(row => row.variant === variant);
|
||||
assert.equal(native?.length, 1, `${variant} qualified native stage is missing`);
|
||||
assert.equal(native[0].binarySha256, descriptor.binarySha256, `${variant} native stage binary mismatch`);
|
||||
assert.equal(receipt.binarySha256, descriptor.binarySha256, `${variant} receipt binary mismatch`);
|
||||
assert.equal(receipt.rustSourceTree, descriptor.rustSourceTree, `${variant} Rust source identity mismatch`);
|
||||
assert.equal(receipt.rustInputObjects?.runner, descriptor.rustSourceTree, `${variant} qualified Rust input objects are missing`);
|
||||
absolute(native[0].targetRealPath, `${variant} qualified native target`);
|
||||
for (const [file, object] of Object.entries(receipt.rustInputObjects)) {
|
||||
assert.ok(file && !path.posix.isAbsolute(file) && path.posix.normalize(file) === file && !file.includes('\\') && !file.split('/').includes('..'), 'Qualified Rust input path is invalid');
|
||||
revision(object, `${variant} Rust input ${file}`);
|
||||
}
|
||||
assert.equal(receipt.artifactInventorySummary, descriptor.qualifiedInventorySummaryReference ?? descriptor.inventorySummaryFile, `${variant} receipt points to another inventory summary`);
|
||||
assert.equal(manifest.variant, variant); assert.equal(manifest.revision, descriptor.revision);
|
||||
assert.equal(manifest.compiledBuildRevision, descriptor.revision); assert.equal(manifest.aggregateSha256, descriptor.aggregateSha256);
|
||||
assert.deepEqual(manifest.roots, INVENTORY_ROOTS);
|
||||
assert.ok(Array.isArray(manifest.files) && manifest.files.length === manifest.fileCount && manifest.fileCount > 0);
|
||||
const row = summary.inventories?.[variant]; assert.ok(row, `${variant} inventory summary entry missing`);
|
||||
assert.equal(row.manifest, path.basename(descriptor.manifestFile)); assert.equal(row.manifestSha256, descriptor.manifestFileSha256);
|
||||
assert.equal(row.aggregateSha256, descriptor.aggregateSha256); assert.equal(row.fileCount, manifest.fileCount);
|
||||
if (Object.hasOwn(BASELINES, variant)) {
|
||||
assert.equal(descriptor.revision, BASELINES[variant], 'Original baseline reference must remain unchanged');
|
||||
assert.equal(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'B0/B1 must use the original, unmodified aligned4a83 qualified receipt');
|
||||
assert.equal(descriptor.inventorySummarySha256, OLD_SUMMARY); assert.equal(descriptor.binarySha256, OLD_BINARY);
|
||||
assert.equal(descriptor.rustSourceTree, OLD_RUST);
|
||||
for (const [key, value] of Object.entries(OLD_INVENTORIES[variant])) assert.equal(descriptor[key], value, `Original ${variant} inventory changed`);
|
||||
} else {
|
||||
assert.equal(variant, 'B2');
|
||||
assert.ok(![...Object.values(BASELINES), OLD_B2].includes(descriptor.revision), 'Final B2 needs its new actual immutable product revision');
|
||||
assert.notEqual(descriptor.qualifiedReceiptSha256, OLD_RECEIPT, 'Final B2 needs its own new qualified receipt');
|
||||
assert.notEqual(descriptor.binarySha256, OLD_BINARY, 'Final Rust B2 needs its newly qualified binary');
|
||||
assert.notEqual(descriptor.rustSourceTree, OLD_RUST, 'Final Rust B2 needs its newly qualified Rust inputs');
|
||||
assert.notEqual(native[0].targetRealPath, OLD_NATIVE_TARGET, 'Final Rust B2 must not reuse the original B0/B1 target');
|
||||
}
|
||||
return { receipt, summary, manifest };
|
||||
}
|
||||
|
||||
// Match the existing inventory generator and H5 helper exactly: Unicode code
|
||||
// point order, Python ensure_ascii JSON with sorted bytes/path/sha256 keys.
|
||||
function pythonOrder(left, right) {
|
||||
const a = [...left], b = [...right];
|
||||
for (let index = 0; index < Math.min(a.length, b.length); index++) {
|
||||
const difference = a[index].codePointAt(0) - b[index].codePointAt(0);
|
||||
if (difference) return difference;
|
||||
}
|
||||
return a.length - b.length;
|
||||
}
|
||||
const pythonRow = row => JSON.stringify({ bytes: row.bytes, path: row.path, sha256: row.sha256 })
|
||||
.replace(/[\u007f-\uffff]/g, character => `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
||||
function inventoryPath(value) {
|
||||
assert.ok(typeof value === 'string' && value.length && !value.includes('\\') && !value.includes('\0')
|
||||
&& !path.posix.isAbsolute(value) && path.posix.normalize(value) === value && !value.split('/').includes('..')
|
||||
&& INVENTORY_ROOTS.some(root => value.startsWith(`${root}/`)), 'Inventory filename escapes registered roots');
|
||||
return value;
|
||||
}
|
||||
export async function validateActualInventory(build, manifest) {
|
||||
const startedAtMs = Date.now(), started = performance.now(), canonicalRoot = await realpath(build.appRoot);
|
||||
assert.deepEqual(manifest.roots, INVENTORY_ROOTS);
|
||||
const expected = new Map(), canonicalFiles = new Set();
|
||||
for (const row of manifest.files) {
|
||||
const name = inventoryPath(row.path);
|
||||
assert.equal(expected.has(name), false, 'Duplicate inventory filename'); expected.set(name, row);
|
||||
digest(row.sha256, 'inventory file'); assert.ok(Number.isSafeInteger(row.bytes) && row.bytes >= 0);
|
||||
}
|
||||
assert.equal(expected.size, manifest.fileCount, 'Inventory file count is incomplete');
|
||||
const actualNames = [], canonicalDirectories = new Set();
|
||||
const walk = async relative => {
|
||||
const directory = path.join(build.appRoot, relative), canonical = await realpath(directory);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory directory resolves outside APP_ROOT');
|
||||
assert.equal(canonicalDirectories.has(canonical), false, 'Inventory directory aliases another registered directory'); canonicalDirectories.add(canonical);
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const child = `${relative}/${entry.name}`;
|
||||
if (entry.isDirectory()) await walk(child);
|
||||
else {
|
||||
const file = path.join(build.appRoot, child), metadata = await lstat(file);
|
||||
if (metadata.isSymbolicLink()) assert.equal((await stat(file)).isDirectory(), false, 'Inventory directory symlink is not a frozen file');
|
||||
assert.ok(metadata.isFile() || metadata.isSymbolicLink(), 'Inventory includes a nonregular entry');
|
||||
inventoryPath(child); actualNames.push(child);
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const root of INVENTORY_ROOTS) await walk(root);
|
||||
actualNames.sort(pythonOrder);
|
||||
assert.deepEqual(actualNames, [...expected.keys()].sort(pythonOrder), 'Actual APP_ROOT file set differs from frozen inventory');
|
||||
const actualRows = new Array(actualNames.length); let next = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(8, actualNames.length) }, async () => {
|
||||
for (;;) {
|
||||
const index = next++; if (index >= actualNames.length) return;
|
||||
const name = actualNames[index], file = path.join(build.appRoot, name), canonical = await realpath(file);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Inventory file resolves outside APP_ROOT');
|
||||
assert.equal(canonicalFiles.has(canonical), false, 'Inventory filenames alias the same canonical file'); canonicalFiles.add(canonical);
|
||||
const before = await stat(file); assert.ok(before.isFile(), 'Inventory evidence must be a regular file');
|
||||
const content = createHash('sha256'); for await (const bytes of createReadStream(file)) content.update(bytes);
|
||||
const after = await stat(file);
|
||||
assert.equal(after.size, before.size, 'Artifact changed during inventory hashing'); assert.equal(after.mtimeMs, before.mtimeMs, 'Artifact changed during inventory hashing');
|
||||
const row = { bytes: after.size, path: name, sha256: content.digest('hex') }, frozen = expected.get(name);
|
||||
assert.equal(row.bytes, frozen.bytes, `Actual artifact size changed: ${name}`); assert.equal(row.sha256, frozen.sha256, `Actual artifact bytes changed: ${name}`);
|
||||
actualRows[index] = row;
|
||||
}
|
||||
}));
|
||||
const aggregateSha256 = hash(Buffer.from(`${actualRows.map(pythonRow).join('\n')}\n`, 'utf8'));
|
||||
assert.equal(aggregateSha256, manifest.aggregateSha256, 'Actual APP_ROOT aggregate differs from frozen inventory');
|
||||
assert.equal(aggregateSha256, build.aggregateSha256, 'Actual APP_ROOT aggregate differs from qualified build');
|
||||
return { appRoot: build.appRoot, canonicalAppRoot: canonicalRoot, fileCount: actualRows.length, aggregateSha256,
|
||||
startedAtMs, completedAtMs: Date.now(), elapsedMs: performance.now() - started,
|
||||
scope: 'Live registered file set, canonical containment/uniqueness, size and SHA checked before this arm; not an assertion of future immutability' };
|
||||
}
|
||||
|
||||
function harnessFields(release) {
|
||||
return Object.fromEntries(['harnessRevision', 'protocolSha256', 'comparatorSha256', 'inventoryHelperSha256', 'protocolDocumentSha256', 'lockSha256', 'gatesSha256', 'workloadSha256', 'observationMode', 'protocolVersion'].map(key => [key, release[key]]));
|
||||
}
|
||||
|
||||
export async function loadRelease(releaseFile, stage, outputRoot) {
|
||||
absolute(releaseFile, 'root release'); absolute(outputRoot, 'output root');
|
||||
const releaseBytes = await readFile(releaseFile), release = JSON.parse(releaseBytes);
|
||||
const plan = stagePlan(stage);
|
||||
assert.equal(release.authorized, true, 'Root release is not authorized'); assert.equal(release.stage, stage);
|
||||
assert.equal(release.outputRoot, outputRoot, 'Root release freezes a different exact output path');
|
||||
assert.equal(release.orchestratorSha256, hash(await readFile(selfFile)), 'Orchestrator changed after release');
|
||||
absolute(release.harnessRoot, 'H5 harness'); revision(release.harnessRevision, 'H5 harness');
|
||||
assert.equal(release.harnessRevision, H5_REVISION, 'Use the actual independently frozen H5 full revision');
|
||||
assert.equal(release.observationMode, MODE); assert.equal(release.protocolVersion, PROTOCOL);
|
||||
assert.equal(release.workloadSha256, WORKLOAD_SHA256);
|
||||
assert.equal(release.resourceWatchdogMillis, RESOURCE_WATCHDOG_MILLIS, 'Resource watchdog registration changed; this is not an SLA');
|
||||
assert.equal(release.lockSha256, LOCK_SHA256); assert.equal(release.gatesSha256, GATES_SHA256);
|
||||
for (const [key, expected] of [['protocolSha256', H5_PROTOCOL_SHA256], ['comparatorSha256', H5_COMPARATOR_SHA256], ['inventoryHelperSha256', H5_INVENTORY_HELPER_SHA256], ['protocolDocumentSha256', H5_PROTOCOL_DOCUMENT_SHA256]])
|
||||
assert.equal(release[key], expected, `Reviewed H5 source pin changed: ${key}`);
|
||||
for (const variant of plan.requiredVariants) await validateVariantBuild(variant, release.perVariantBuilds?.[variant]);
|
||||
const context = { releaseFile, releaseBytesSha256: hash(releaseBytes), release, plan, out: outputRoot, commands: [], commandIndex: 0 };
|
||||
await checkPriorStages(context);
|
||||
if (plan.noiseStage) await checkNoise(context);
|
||||
if (plan.finalizationOnly) await checkFinalizationInputs(context);
|
||||
return context;
|
||||
}
|
||||
|
||||
async function checkReleaseBytes(context) {
|
||||
assert.equal(hash(await readFile(context.releaseFile)), context.releaseBytesSha256, 'Exact root release bytes changed');
|
||||
assert.equal(hash(await readFile(selfFile)), context.release.orchestratorSha256, 'Orchestrator changed after release');
|
||||
if (context.rootReleaseCopy) assert.equal(hash(await readFile(context.rootReleaseCopy)), context.releaseBytesSha256, 'Preserved root release bytes changed');
|
||||
}
|
||||
|
||||
function checkComparisonIdentity(report, context) {
|
||||
assert.equal(report.complete, true); assert.equal(report.observationMode, MODE); assert.equal(report.protocolVersion, PROTOCOL);
|
||||
assert.equal(report.workloadSha256, WORKLOAD_SHA256); assert.equal(report.measurementSha256, context.release.protocolSha256);
|
||||
assert.equal(report.comparatorSha256, context.release.comparatorSha256); assert.equal(report.gatesSha256, GATES_SHA256);
|
||||
assert.equal(report.pairCount, 5);
|
||||
}
|
||||
|
||||
async function checkPriorStages(context) {
|
||||
for (const stage of context.plan.priorStages) {
|
||||
const pin = context.release.priorStageReports?.[stage]; assert.ok(pin, `Stage order requires frozen ${stage} first`);
|
||||
assert.equal(pin.file, path.join(context.out, `${stage}.json`), 'Prior stage report path changed');
|
||||
const report = (await hashedJson(pin.file, pin.sha256, stage)).value;
|
||||
checkComparisonIdentity(report, context);
|
||||
const priorPlan = stagePlan(stage);
|
||||
assert.equal(report.phase, priorPlan.aa ? 'aa' : 'compare');
|
||||
assert.equal(report.baselineRevision, BASELINES[priorPlan.baseline]);
|
||||
const candidateRevision = BASELINES[priorPlan.candidate] ?? context.release.perVariantBuilds.B2?.revision;
|
||||
assert.equal(report.candidateRevision, candidateRevision, 'Prior stage used a different measured candidate revision');
|
||||
for (const [arm, variant] of [['baseline', priorPlan.baseline], ['candidate', priorPlan.candidate]]) {
|
||||
const expected = context.release.perVariantBuilds?.[variant] ?? (BASELINES[variant] ? { ...OLD_INVENTORIES[variant], qualifiedReceiptSha256: OLD_RECEIPT, inventorySummarySha256: OLD_SUMMARY } : undefined);
|
||||
assert.ok(expected, 'Prior candidate qualified build is unavailable');
|
||||
for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256'])
|
||||
assert.equal(report.artifactInventories?.[arm]?.[key], expected[key], 'Prior stage changed a measured qualified inventory');
|
||||
}
|
||||
if (!priorPlan.aa) assert.equal(report.performancePassed, true, `Prior ${stage} performance failed`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function checkNoise(context) {
|
||||
const { release, plan, out } = context;
|
||||
const expected = path.join(out, `${plan.noiseStage}.json`);
|
||||
assert.equal(release.noiseReportFile, expected, 'Use the same exact frozen A/A noise path');
|
||||
const noise = (await hashedJson(expected, release.noiseReportSha256, 'Reviewed A/A noise report')).value;
|
||||
checkComparisonIdentity(noise, context); assert.equal(noise.phase, 'aa');
|
||||
assert.equal(noise.baselineRevision, release.perVariantBuilds[plan.baseline].revision);
|
||||
assert.equal(noise.candidateRevision, noise.baselineRevision);
|
||||
assert.equal(noise.h5Assessments?.length, 5, 'Reviewed A/A must contain all five complete assessments');
|
||||
assert.ok(noise.measurements?.length > 0 && noise.measurements.every(row => Number.isFinite(row.aaNoiseEnvelopeRelative) && row.aaNoiseEnvelopeRelative >= 0), 'A/A noise envelope is invalid');
|
||||
const build = release.perVariantBuilds[plan.baseline];
|
||||
for (const arm of ['baseline', 'candidate']) for (const key of ['manifestFileSha256', 'aggregateSha256', 'qualifiedReceiptSha256', 'inventorySummarySha256'])
|
||||
assert.equal(noise.artifactInventories?.[arm]?.[key], build[key], 'Frozen A/A baseline inventory differs from required build');
|
||||
return noise;
|
||||
}
|
||||
|
||||
const QUIET_SETTLE_MILLIS = 30_000, QUIET_POLL_MILLIS = 1_000, QUIET_PS_TIMEOUT_MILLIS = 2_000;
|
||||
const QUIET_SCHEMA = 'paperclip.native-ingestion.physical-quiet.v1';
|
||||
const QUIET_SCOPE = 'Dedicated Linux process-name guard for specified Node/PostgreSQL/Runner/browser/Rust-build names only; not a claim that every possible host activity is absent. Qualified H5 Stop proof separately covers provider/sleep control groups.';
|
||||
const QUIET_ZOMBIE_POLICY = 'Strict: matched zombie/dead rows remain blockers until absent; no automatic killing or silent zombie exclusion.';
|
||||
function processFamily(comm) {
|
||||
if (/^(node|node-MainThread)$/.test(comm)) return 'node';
|
||||
if (/^(postgres|postmaster|pg_ctl|initdb)$/.test(comm)) return 'postgresql';
|
||||
if (/^(paperclip-runne|paperclip-runner.*|Runner)$/.test(comm)) return 'runner';
|
||||
if (/^(chrome|headless|chromium)/i.test(comm)) return 'browser';
|
||||
if (/^(rustc|cargo)$/.test(comm)) return 'rust-build';
|
||||
return null;
|
||||
}
|
||||
export function classifyLinuxProcesses(text, orchestratorPid = process.pid) {
|
||||
assert.ok(typeof text === 'string' && text.trim(), 'Linux ps returned no process rows');
|
||||
assert.ok(Number.isSafeInteger(orchestratorPid) && orchestratorPid > 0, 'Invalid orchestrator PID');
|
||||
const seen = new Set(), matched = []; let rowCount = 0;
|
||||
for (const line of text.trim().split(/\r?\n/)) {
|
||||
const fields = line.match(/^\s*(\d+)\s+(\d+)\s+(.+?)\s+([DIRSTtWXZPK])\s*$/);
|
||||
assert.ok(fields, 'Malformed Linux ps PID/PPID/comm/state row');
|
||||
const row = { pid: Number(fields[1]), ppid: Number(fields[2]), comm: fields[3].trim(), state: fields[4] };
|
||||
assert.ok(Number.isSafeInteger(row.pid) && row.pid > 0 && Number.isSafeInteger(row.ppid) && row.ppid >= 0 && row.comm.length > 0 && row.comm.length <= 256 && !row.comm.includes('\0'), 'Malformed Linux ps process fields');
|
||||
assert.equal(seen.has(row.pid), false, 'Duplicate Linux ps PID'); seen.add(row.pid); rowCount++;
|
||||
const family = processFamily(row.comm);
|
||||
if (family) matched.push({ ...row, family, allowed: row.pid === orchestratorPid && !['Z', 'X'].includes(row.state) });
|
||||
}
|
||||
assert.ok(seen.has(orchestratorPid), 'Linux ps omitted the current orchestrator PID');
|
||||
return { rowCount, matched, blockers: matched.filter(row => !row.allowed) };
|
||||
}
|
||||
function linuxProcessRows(timeoutMillis) {
|
||||
return execFileSync('/bin/ps', ['-eo', 'pid=,ppid=,comm=,state='], {
|
||||
encoding: 'utf8', timeout: timeoutMillis, maxBuffer: 2 * 1024 * 1024, env: { LC_ALL: 'C' },
|
||||
});
|
||||
}
|
||||
export async function physicalQuietGuard({ receiptFile, phase, armStage, primaryFailureAlreadyPresent = false,
|
||||
platform = process.platform, orchestratorPid = process.pid, readPs = linuxProcessRows,
|
||||
now = () => performance.now(), pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)) }) {
|
||||
absolute(receiptFile, 'physical quiet receipt');
|
||||
const started = now(), receipt = { schema: QUIET_SCHEMA, scope: QUIET_SCOPE, phase, armStage, platform, orchestratorPid,
|
||||
fields: ['pid', 'ppid', 'comm', 'state'], zombiePolicy: QUIET_ZOMBIE_POLICY,
|
||||
resourceSettleLimitMillis: QUIET_SETTLE_MILLIS, pollIntervalMillis: QUIET_POLL_MILLIS, psTimeoutMillis: QUIET_PS_TIMEOUT_MILLIS,
|
||||
primaryFailureAlreadyPresent, startedAtMs: Date.now(), passed: false, observations: [] };
|
||||
let failure;
|
||||
try {
|
||||
assert.equal(platform, 'linux', 'Accepted H5 physical quiet guard requires Linux');
|
||||
assert.ok(['pre-arm', 'post-arm'].includes(phase), 'Unknown physical quiet phase');
|
||||
for (;;) {
|
||||
const remaining = QUIET_SETTLE_MILLIS - (now() - started);
|
||||
assert.ok(remaining > 0, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
const observed = classifyLinuxProcesses(readPs(Math.max(1, Math.min(QUIET_PS_TIMEOUT_MILLIS, Math.floor(remaining)))), orchestratorPid);
|
||||
receipt.observations.push({ atMs: Date.now(), elapsedMillis: now() - started, ...observed });
|
||||
assert.ok(now() - started < QUIET_SETTLE_MILLIS, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
if (observed.blockers.length === 0) { receipt.passed = true; break; }
|
||||
const wait = Math.min(QUIET_POLL_MILLIS, QUIET_SETTLE_MILLIS - (now() - started));
|
||||
assert.ok(wait > 0, 'Physical quiet settle deadline expired; stop for cleanup');
|
||||
await pause(wait);
|
||||
}
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
receipt.failure = { name: error instanceof Error ? error.name : 'UnknownError', message: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
receipt.completedAtMs = Date.now(); receipt.elapsedMillis = now() - started;
|
||||
const bytes = Buffer.from(JSON.stringify(receipt, null, 2) + '\n');
|
||||
await writeFile(receiptFile, bytes, { flag: 'wx' });
|
||||
const result = { file: receiptFile, bytes: bytes.length, sha256: hash(bytes), passed: receipt.passed };
|
||||
if (failure) {
|
||||
const error = new Error(`Physical quiet guard failed (${phase}); receipt=${receiptFile}; sha256=${result.sha256}`, { cause: failure });
|
||||
Object.assign(error, { receiptFile, receiptSha256: result.sha256 }); throw error;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
export async function assertNoPreferredUiShadow(appRoot, statPath = lstat) {
|
||||
const preferredIndex = path.join(appRoot, 'server/ui-dist/index.html'), preferredDirectory = path.dirname(preferredIndex);
|
||||
try { await statPath(preferredIndex); assert.fail('Preferred server/ui-dist/index.html exists (including a symlink); ui/dist inventory cannot attest the selected UI'); }
|
||||
catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
try { assert.equal((await statPath(preferredDirectory)).isSymbolicLink(), false, 'Preferred server/ui-dist directory symlink prevents the absence guard'); }
|
||||
catch (error) { if (error.code !== 'ENOENT') throw error; }
|
||||
return { checkedAtMs: Date.now(), preferredIndex, preferredIndexAbsent: true, preferredDirectorySymlink: false,
|
||||
selectedUiRoot: path.join(appRoot, 'ui/dist'),
|
||||
assumption: 'Preferred server/ui-dist index absent at preflight; inventoried ui/dist is the compiled fallback. No routing/build changes or future immutability claim.' };
|
||||
}
|
||||
|
||||
async function checkFrozen(context, selectedVariant) {
|
||||
await checkReleaseBytes(context);
|
||||
for (const key of ['DATABASE_URL', 'PAPERCLIP_RUNNER_STATE_DIR', 'PAPERCLIP_INGESTION_SERVER_COMMAND', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_DIR',
|
||||
'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_FS', 'PAPERCLIP_NATIVE_INGESTION_DIAGNOSTIC_INSPECTOR', 'PAPERCLIP_INGESTION_BROWSER_CAPTURE', 'NODE_OPTIONS'])
|
||||
assert.ok(!process.env[key], `Acceptance rejects ambient ${key}`);
|
||||
const { release, plan } = context, actualInventories = {};
|
||||
for (const variant of plan.requiredVariants) {
|
||||
const build = release.perVariantBuilds[variant], qualified = await validateVariantBuild(variant, build);
|
||||
assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.revision, `${variant} SUT revision changed`);
|
||||
assert.equal(execFileSync('git', ['status', '--short'], { cwd: build.appRoot, encoding: 'utf8' }).trim(), '', `${variant} SUT is dirty`);
|
||||
assert.equal((await getJson(path.join(build.appRoot, 'server/dist/build-info.json'))).commit, build.revision);
|
||||
const binaryFile = path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd');
|
||||
const native = qualified.receipt.nativeStageHashes.find(row => row.variant === variant);
|
||||
const actualTarget = await realpath(path.join(build.appRoot, 'packages/paperclip-runner/runner/target'));
|
||||
assert.equal(actualTarget, native.targetRealPath, `${variant} actual native target differs from qualified receipt`);
|
||||
assert.equal(await realpath(binaryFile), path.join(actualTarget, 'release/paperclip-runnerd'), `${variant} native binary resolves outside its exact qualified target`);
|
||||
assert.equal(hash(await readFile(binaryFile)), build.binarySha256, `${variant} actual native binary changed`);
|
||||
for (const [file, object] of Object.entries(qualified.receipt.rustInputObjects))
|
||||
assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/${file}`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), object, `${variant} qualified Rust input changed: ${file}`);
|
||||
assert.equal(execFileSync('git', ['rev-parse', `${build.revision}:packages/paperclip-runner/runner`], { cwd: build.appRoot, encoding: 'utf8' }).trim(), build.rustSourceTree);
|
||||
if (!selectedVariant || selectedVariant === variant) actualInventories[variant] = {
|
||||
...await validateActualInventory(build, qualified.manifest), uiServing: await assertNoPreferredUiShadow(build.appRoot),
|
||||
};
|
||||
}
|
||||
assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), release.harnessRevision);
|
||||
assert.equal(execFileSync('git', ['status', '--short'], { cwd: release.harnessRoot, encoding: 'utf8' }).trim(), '');
|
||||
assert.equal(hash(Buffer.concat(await Promise.all(protocolFiles.map(file => readFile(path.join(release.harnessRoot, 'tests/e2e', file)))))), release.protocolSha256);
|
||||
for (const [file, expected] of [['scripts/compare-native-ingestion-browser.ts', release.comparatorSha256], ['scripts/native-ingestion-h5-inventory.mjs', release.inventoryHelperSha256], ['tests/e2e/fixtures/native-ingestion-h5.md', release.protocolDocumentSha256], ['scripts/native-ingestion-performance-gates.mjs', release.gatesSha256], ['pnpm-lock.yaml', release.lockSha256]])
|
||||
assert.equal(hash(await readFile(path.join(release.harnessRoot, file))), expected, `Frozen H5 support changed: ${file}`);
|
||||
await checkPriorStages(context);
|
||||
if (plan.noiseStage) await checkNoise(context);
|
||||
return { checkedAtMs: Date.now(), releaseFileSha256: context.releaseBytesSha256, actualInventories };
|
||||
}
|
||||
|
||||
async function execute(context, step) {
|
||||
const index = ++context.commandIndex, startedAt = new Date().toISOString(), log = path.join(context.out, `${step.stage}.log`);
|
||||
const command = { index, stage: step.stage, cwd: step.cwd, executable: process.execPath, args: step.args, environmentOverrides: step.env ?? {}, startedAt, log,
|
||||
resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS, resourceWatchdogScope: 'Resource-only cap including startup/test/teardown; original SLA and H5 observation limits remain driver-owned' };
|
||||
context.commands.push(command);
|
||||
const save = () => saveJson(path.join(context.out, `${context.plan.stage}-commands.json`), context.commands);
|
||||
await save(); console.log(`START ${step.stage} ${startedAt}; log=${log}`);
|
||||
const stream = (await open(log, 'wx')).createWriteStream();
|
||||
const child = spawn(process.execPath, step.args, { cwd: step.cwd, env: { ...process.env, ...step.env }, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
command.childPid = child.pid; await save();
|
||||
const interrupt = () => child.kill('SIGINT'), terminate = () => child.kill('SIGTERM');
|
||||
process.once('SIGINT', interrupt); process.once('SIGTERM', terminate);
|
||||
let forcedStop;
|
||||
const watchdog = setTimeout(() => {
|
||||
command.watchdogExpired = true; child.kill('SIGTERM');
|
||||
forcedStop = setTimeout(() => { command.forcedDriverKill = true; child.kill('SIGKILL'); }, 30_000);
|
||||
}, RESOURCE_WATCHDOG_MILLIS);
|
||||
child.stdout.pipe(stream, { end: false }); child.stderr.pipe(stream, { end: false });
|
||||
let exit;
|
||||
try { exit = await new Promise((resolve, reject) => { child.once('error', reject); child.once('close', (code, signal) => resolve({ code, signal })); }); }
|
||||
finally { clearTimeout(watchdog); clearTimeout(forcedStop); process.off('SIGINT', interrupt); process.off('SIGTERM', terminate); await new Promise(resolve => stream.end(resolve)); }
|
||||
Object.assign(command, { finishedAt: new Date().toISOString(), ...exit }); await save();
|
||||
console.log(`END ${step.stage} code=${exit.code} signal=${exit.signal ?? 'none'}`);
|
||||
assert.equal(command.watchdogExpired, undefined, 'Resource watchdog expired; stop for physical cleanup, never accept or advance');
|
||||
assert.equal(exit.code, 0, `Stage ${step.stage} failed; retain raw files and perform physical cleanup before continuing`);
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
|
||||
const port = server.address().port; await new Promise(resolve => server.close(resolve)); return port;
|
||||
}
|
||||
async function assertPortReleased(port) {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
}
|
||||
async function extractEvidence(directory) {
|
||||
const playwrightFile = path.join(directory, 'playwright.json'), report = await getJson(playwrightFile), attachments = [];
|
||||
const visit = node => { if (Array.isArray(node)) for (const child of node) visit(child); else if (node && typeof node === 'object') {
|
||||
if (Array.isArray(node.attachments)) attachments.push(...node.attachments.filter(value => value.name === 'native-ingestion-evidence'));
|
||||
for (const [key, child] of Object.entries(node)) if (key !== 'attachments') visit(child);
|
||||
} };
|
||||
visit(report); assert.equal(attachments.length, 1, 'Exactly one native-ingestion evidence attachment is required');
|
||||
const attachment = attachments[0];
|
||||
const evidence = attachment.body ? JSON.parse(Buffer.from(attachment.body, 'base64').toString()) : await getJson(path.resolve(directory, attachment.path));
|
||||
const file = path.join(directory, 'native-ingestion-evidence.json'); await saveJson(file, evidence, true);
|
||||
return { file, evidence, playwrightFile };
|
||||
}
|
||||
|
||||
async function armSidecar(directory, variant, build) {
|
||||
const destination = path.join(directory, 'inventory'); await mkdir(destination);
|
||||
const copies = {};
|
||||
for (const [key, sha] of [['manifestFile', 'manifestFileSha256'], ['inventorySummaryFile', 'inventorySummarySha256'], ['qualifiedReceiptFile', 'qualifiedReceiptSha256']]) {
|
||||
const bytes = await readFile(build[key]); assert.equal(hash(bytes), build[sha], 'Per-arm provenance changed while copying');
|
||||
const file = path.join(destination, path.basename(build[key])); await writeFile(file, bytes, { flag: 'wx' }); copies[key] = path.relative(directory, file);
|
||||
}
|
||||
const sidecar = { schema: 'paperclip.native-ingestion.arm-inventory.v1', variant, sourceRevision: build.revision, compiledBuildRevision: build.revision,
|
||||
playwrightReportSha256: hash(await readFile(path.join(directory, 'playwright.json'))), ...copies,
|
||||
manifestFileSha256: build.manifestFileSha256, aggregateSha256: build.aggregateSha256,
|
||||
qualifiedReceiptSha256: build.qualifiedReceiptSha256, inventorySummarySha256: build.inventorySummarySha256 };
|
||||
const file = path.join(directory, 'arm-inventory.json'); await saveJson(file, sidecar, true); return file;
|
||||
}
|
||||
|
||||
async function armArtifactFiles(result) {
|
||||
const sidecar = await getJson(result.inventoryFile), base = path.dirname(result.inventoryFile);
|
||||
return [result.file, result.playwrightFile, result.inventoryFile,
|
||||
...['manifestFile', 'inventorySummaryFile', 'qualifiedReceiptFile'].map(key => path.resolve(base, sidecar[key])),
|
||||
path.join(base, 'pre-arm-inventory-check.json'), path.join(base, 'physical-http-port-check.json'),
|
||||
path.join(base, 'physical-quiet-pre.json'), path.join(base, 'physical-quiet-post.json')];
|
||||
}
|
||||
export async function sealArmArtifacts(out, result) {
|
||||
const canonicalRoot = await realpath(out), files = await armArtifactFiles(result), seen = new Set(), seals = [];
|
||||
for (const file of files) {
|
||||
absolute(file, 'raw arm artifact'); const canonical = await realpath(file);
|
||||
assert.ok(canonical.startsWith(`${canonicalRoot}${path.sep}`), 'Raw arm artifact resolves outside owned output');
|
||||
assert.equal(seen.has(canonical), false, 'Duplicate raw arm artifact'); seen.add(canonical);
|
||||
const bytes = await readFile(file); seals.push({ file, bytes: bytes.length, sha256: hash(bytes) });
|
||||
}
|
||||
return seals;
|
||||
}
|
||||
export async function verifyRawArtifactSeals(out, manifest) {
|
||||
const canonicalRoot = await realpath(out);
|
||||
for (const entry of manifest) for (const arm of ['baseline', 'candidate']) {
|
||||
const seals = entry[`${arm}Artifacts`]; assert.ok(Array.isArray(seals) && seals.length === 10, 'Raw arm byte seals are incomplete');
|
||||
const seen = new Set();
|
||||
for (const seal of seals) {
|
||||
absolute(seal.file, 'sealed arm artifact'); digest(seal.sha256, 'sealed arm artifact');
|
||||
assert.equal(seen.has(seal.file), false, 'Duplicate sealed arm artifact'); seen.add(seal.file);
|
||||
assert.ok((await realpath(seal.file)).startsWith(`${canonicalRoot}${path.sep}`), 'Sealed arm artifact resolves outside output');
|
||||
const bytes = await readFile(seal.file);
|
||||
assert.equal(bytes.length, seal.bytes, 'Raw arm artifact size changed after timing');
|
||||
assert.equal(hash(bytes), seal.sha256, 'Raw arm artifact bytes changed after timing');
|
||||
}
|
||||
const expected = await armArtifactFiles({ file: entry[`${arm}File`], playwrightFile: entry[`${arm}PlaywrightFile`], inventoryFile: entry[`${arm}InventoryFile`] });
|
||||
assert.deepEqual([...seen].sort(), expected.sort(), 'Raw seals do not cover this exact evidence/PW/sidecar/inventory set');
|
||||
for (const [filename, phase] of [['physical-quiet-pre.json', 'pre-arm'], ['physical-quiet-post.json', 'post-arm']]) {
|
||||
const receipt = await getJson(path.join(path.dirname(entry[`${arm}InventoryFile`]), filename));
|
||||
assert.equal(receipt.schema, QUIET_SCHEMA); assert.equal(receipt.passed, true); assert.equal(receipt.platform, 'linux');
|
||||
assert.equal(receipt.phase, phase); assert.equal(receipt.zombiePolicy, QUIET_ZOMBIE_POLICY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function physicalGuardReferences(manifest) {
|
||||
return manifest.flatMap((entry, pair) => ['baseline', 'candidate'].map(arm => ({ pair: pair + 1, arm,
|
||||
receipts: entry[`${arm}Artifacts`].filter(seal => /^physical-quiet-(pre|post)\.json$/.test(path.basename(seal.file))) })));
|
||||
}
|
||||
|
||||
async function browserRun(context, armStage, variant) {
|
||||
const preflight = await checkFrozen(context, variant); // ALL receipts; selected live artifact inventory before EVERY arm.
|
||||
const { release } = context, build = release.perVariantBuilds[variant], directory = path.join(context.out, armStage);
|
||||
await mkdir(directory, { recursive: false }); await saveJson(path.join(directory, 'pre-arm-inventory-check.json'), preflight, true);
|
||||
await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-pre.json'), phase: 'pre-arm', armStage });
|
||||
const port = await freePort();
|
||||
let primaryError, inventoryFile, extracted;
|
||||
try {
|
||||
await execute(context, { stage: armStage, cwd: release.harnessRoot,
|
||||
args: ['node_modules/@playwright/test/cli.js', 'test', '--config', 'tests/e2e/playwright-native-ingestion.config.ts'],
|
||||
env: { PAPERCLIP_INGESTION_OBSERVATION_MODE: MODE, PAPERCLIP_RUNNER_BINARY: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'),
|
||||
PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: path.join(build.appRoot, 'packages/paperclip-runner/runner/target/release/paperclip-runnerd'),
|
||||
PAPERCLIP_INGESTION_TRANSPORT: 'sandbox-remote-with-local-stop', PAPERCLIP_E2E_PORT: String(port), PAPERCLIP_INGESTION_SMOKE: '0',
|
||||
PAPERCLIP_INGESTION_VARIANT: `${armStage}-${variant}`, PAPERCLIP_INGESTION_APP_ROOT: build.appRoot, PAPERCLIP_INGESTION_EVIDENCE_DIR: directory,
|
||||
PAPERCLIP_INGESTION_DRIVER_DEPENDENCIES: 'Independent frozen H5 driver install; each measured variant owns its separately qualified application/native build' } });
|
||||
} catch (error) { primaryError = error; }
|
||||
try {
|
||||
inventoryFile = await armSidecar(directory, variant, build); extracted = await extractEvidence(directory);
|
||||
if (!primaryError) {
|
||||
const { evidence } = extracted;
|
||||
assert.equal(evidence.complete, true); assert.equal(evidence.observationComplete, true); assert.equal(evidence.observationMode, MODE);
|
||||
assert.equal(evidence.workloadSha256, WORKLOAD_SHA256); assert.equal(evidence.protocolVersion, PROTOCOL);
|
||||
assert.equal(evidence.transport, 'sandbox-remote-with-local-stop'); assert.equal(evidence.cleanup?.complete, true); assert.deepEqual(evidence.cleanup.errors, []);
|
||||
assert.equal(evidence.measurementSha256, release.protocolSha256); assert.equal(evidence.harnessProvenance?.sourceRevision, release.harnessRevision); assert.equal(evidence.harnessProvenance?.sourceStatus, '');
|
||||
assert.equal(evidence.sourceRevision, build.revision); assert.equal(evidence.compiledBuildRevision, build.revision);
|
||||
assert.equal(evidence.serverArtifact, 'compiled'); assert.equal(evidence.qualificationOnly, false); assert.equal(evidence.diagnosticOnly, false); assert.equal(evidence.performanceAcceptanceEligible, true);
|
||||
assert.equal(evidence.browserRecording?.trace, 'off'); assert.equal(evidence.browserRecording?.video, 'off'); assert.equal(evidence.browserRecording?.screencast, false);
|
||||
const loaded = evidence.cases.find(item => item.name === 'burst512'); assert.equal(loaded.runs, 100); assert.equal(loaded.participants, 100); assert.equal(loaded.durable.length, 100);
|
||||
const { validateArmInventory } = await import(pathToFileURL(path.join(release.harnessRoot, 'scripts/native-ingestion-h5-inventory.mjs')).href);
|
||||
await validateArmInventory(inventoryFile, { evidence, owningPlaywright: { reportSha256: hash(await readFile(extracted.playwrightFile)) } });
|
||||
}
|
||||
} catch (error) { primaryError ??= error; }
|
||||
try { await assertPortReleased(port); await saveJson(path.join(directory, 'physical-http-port-check.json'), { released: true, port, scope: 'Owned HTTP listener only; PG/Runner/provider physical cleanup is separately required before the next release' }, true); }
|
||||
catch (error) { primaryError ??= error; }
|
||||
try { await physicalQuietGuard({ receiptFile: path.join(directory, 'physical-quiet-post.json'), phase: 'post-arm', armStage,
|
||||
primaryFailureAlreadyPresent: Boolean(primaryError) }); }
|
||||
catch (error) {
|
||||
if (primaryError) console.error(`Secondary physical quiet failure; primary fixture failure is retained: ${error.message}`);
|
||||
primaryError ??= error;
|
||||
} // Failed guard receipt and SHA remain independent evidence; the fixture failure stays primary.
|
||||
if (primaryError) throw primaryError;
|
||||
const result = { file: extracted.file, playwrightFile: extracted.playwrightFile, inventoryFile };
|
||||
return { ...result, artifacts: await sealArmArtifacts(context.out, result) };
|
||||
}
|
||||
|
||||
function comparatorArgs(context, plan, pairsFile, output, originalProof) {
|
||||
const args = ['scripts/compare-native-ingestion-browser.ts', `--observation-mode=${MODE}`, `--phase=${plan.aa ? 'aa' : 'compare'}`,
|
||||
`--expect=${plan.expectation}`, `--pairs=${pairsFile}`, `--output=${output}`];
|
||||
if (!plan.aa) args.push(`--candidate-policy=${plan.candidatePolicy}`, `--noise-report=${path.join(context.out, `${plan.noiseStage}.json`)}`,
|
||||
...(originalProof ? [`--original-h4e-proof=${originalProof.proofFile}`, `--original-h4e-inventory=${originalProof.inventoryFile}`, '--require-acceptance'] : ['--require-performance']));
|
||||
return args;
|
||||
}
|
||||
|
||||
async function browserSeries(context) {
|
||||
const { plan, release } = context, manifest = [], file = path.join(context.out, `${plan.stage}-pairs.json`), comparisonFile = path.join(context.out, `${plan.stage}.json`);
|
||||
await access(comparisonFile).then(() => assert.fail('Prior comparison output already exists'), error => { if (error.code !== 'ENOENT') throw error; });
|
||||
await saveJson(file, manifest, true);
|
||||
for (let pair = 0; pair < 5; pair++) {
|
||||
const entry = {}, arms = pair % 2 ? [['candidate', plan.candidate], ['baseline', plan.baseline]] : [['baseline', plan.baseline], ['candidate', plan.candidate]];
|
||||
for (const [arm, variant] of arms) {
|
||||
const result = await browserRun(context, `${plan.stage}-pair${pair + 1}-${arm}`, variant);
|
||||
entry[`${arm}File`] = result.file; entry[`${arm}PlaywrightFile`] = result.playwrightFile; entry[`${arm}InventoryFile`] = result.inventoryFile; entry[`${arm}Artifacts`] = result.artifacts;
|
||||
}
|
||||
manifest.push(entry); await saveJson(file, manifest);
|
||||
}
|
||||
await checkFrozen(context); // Same exact release/noise path and bytes pre-comparator.
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
const manifestSha256 = hash(await readFile(file));
|
||||
await execute(context, { stage: `${plan.stage}-comparison`, cwd: release.harnessRoot, args: comparatorArgs(context, plan, file, comparisonFile) });
|
||||
assert.equal(hash(await readFile(file)), manifestSha256, 'Raw paired manifest changed during comparison');
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
const result = await getJson(comparisonFile); checkComparisonIdentity(result, context);
|
||||
if (!plan.aa) assert.equal(result.performancePassed, true);
|
||||
await saveJson(path.join(context.out, `${plan.stage}.completion.json`), { stage: plan.stage, complete: true, fullArms: 10, completedAtMs: Date.now(),
|
||||
releaseFile: context.rootReleaseCopy, originalReleaseFile: context.releaseFile, releaseFileSha256: context.releaseBytesSha256, harnessPins: harnessFields(release),
|
||||
perVariantBuilds: Object.fromEntries(plan.requiredVariants.map(variant => [variant, release.perVariantBuilds[variant]])),
|
||||
manifestFile: file, manifestSha256, comparisonFile, comparisonSha256: hash(await readFile(comparisonFile)),
|
||||
physicalGuards: physicalGuardReferences(manifest),
|
||||
...(plan.noiseStage ? { noiseReportFile: release.noiseReportFile, noiseReportSha256: release.noiseReportSha256 } : {}),
|
||||
performancePassed: result.performancePassed ?? null, acceptancePassed: result.acceptancePassed ?? null,
|
||||
finalOriginalH4eConfirmationPending: plan.candidatePolicy === 'final-b2' }, true);
|
||||
}
|
||||
|
||||
async function checkFinalizationInputs(context) {
|
||||
const proof = context.release.originalH4eProof;
|
||||
assert.ok(proof, 'One later unchanged H4e proof is required for both final comparisons');
|
||||
for (const [key, sha] of [['proofFile', 'proofSha256'], ['inventoryFile', 'inventorySha256']]) await hashedJson(proof[key], proof[sha], `Original H4e ${key}`);
|
||||
const proofReport = await getJson(proof.proofFile), proofAttempts = [];
|
||||
const visit = value => {
|
||||
if (Array.isArray(value)) for (const child of value) visit(child);
|
||||
else if (value && typeof value === 'object') {
|
||||
if (Array.isArray(value.results) && Object.hasOwn(value, 'expectedStatus')) proofAttempts.push(...value.results);
|
||||
for (const [key, child] of Object.entries(value)) if (key !== 'results') visit(child);
|
||||
}
|
||||
};
|
||||
visit(proofReport); assert.equal(proofAttempts.length, 1, 'One later unchanged original H4e attempt must confirm both stages');
|
||||
const proofStartedAtMs = Date.parse(proofAttempts[0].startTime);
|
||||
assert.ok(Number.isFinite(proofStartedAtMs), 'Original H4e proof has no actual attempt start time');
|
||||
for (const stage of ['browser-b1-b2', 'browser-b0-b2']) {
|
||||
const pin = context.release.completedSeries?.[stage]; assert.ok(pin, `Missing frozen completed ${stage}`);
|
||||
assert.equal(pin.file, path.join(context.out, `${stage}.completion.json`));
|
||||
const completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value;
|
||||
assert.equal(completion.stage, stage); assert.equal(completion.complete, true); assert.equal(completion.fullArms, 10); assert.equal(completion.performancePassed, true);
|
||||
assert.ok(Number.isFinite(completion.completedAtMs) && completion.completedAtMs < proofStartedAtMs, 'Original H4e proof must run after BOTH timing stages complete');
|
||||
assert.deepEqual(completion.harnessPins, harnessFields(context.release));
|
||||
const plan = stagePlan(stage);
|
||||
for (const variant of plan.requiredVariants) assert.deepEqual(completion.perVariantBuilds[variant], context.release.perVariantBuilds[variant], 'Finalization changed a measured qualified build');
|
||||
assert.equal(completion.manifestFile, path.join(context.out, `${stage}-pairs.json`));
|
||||
assert.equal(completion.comparisonFile, path.join(context.out, `${stage}.json`));
|
||||
assert.equal(hash(await readFile(completion.manifestFile)), completion.manifestSha256, 'Raw timing manifest changed before finalization');
|
||||
const manifest = await getJson(completion.manifestFile); assert.equal(manifest.length, 5);
|
||||
await verifyRawArtifactSeals(context.out, manifest);
|
||||
assert.deepEqual(completion.physicalGuards, physicalGuardReferences(manifest), 'Completion physical-guard receipts differ from sealed arm bytes');
|
||||
assert.equal(hash(await readFile(completion.comparisonFile)), completion.comparisonSha256, 'Preliminary performance assessment changed');
|
||||
assert.equal(completion.noiseReportFile, path.join(context.out, `${plan.noiseStage}.json`), 'Finalization rebound the frozen noise path');
|
||||
assert.equal(hash(await readFile(completion.noiseReportFile)), completion.noiseReportSha256, 'Frozen noise bytes changed before finalization');
|
||||
const originalRelease = await hashedJson(completion.releaseFile, completion.releaseFileSha256, `${stage} original release`);
|
||||
assert.equal(originalRelease.value.noiseReportFile, completion.noiseReportFile); assert.equal(originalRelease.value.noiseReportSha256, completion.noiseReportSha256);
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeSeries(context) {
|
||||
await checkFrozen(context); await checkFinalizationInputs(context);
|
||||
for (const stage of ['browser-b1-b2', 'browser-b0-b2']) {
|
||||
await checkFrozen(context); await checkFinalizationInputs(context);
|
||||
const pin = context.release.completedSeries[stage], completion = (await hashedJson(pin.file, pin.sha256, `${stage} completion`)).value;
|
||||
const output = path.join(context.out, `${stage}.final.json`);
|
||||
await access(output).then(() => assert.fail('Final assessment output already exists'), error => { if (error.code !== 'ENOENT') throw error; });
|
||||
await execute(context, { stage: `${stage}-original-h4e-finalization`, cwd: context.release.harnessRoot,
|
||||
args: comparatorArgs(context, stagePlan(stage), completion.manifestFile, output, context.release.originalH4eProof) });
|
||||
await checkFinalizationInputs(context);
|
||||
assert.equal((await getJson(output)).acceptancePassed, true);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const stage = option('stage');
|
||||
if (!process.argv.includes('--execute') && !process.argv.includes('--validate-release')) {
|
||||
console.log(JSON.stringify({ draft: true, observationMode: MODE, protocolVersion: PROTOCOL, workloadSha256: WORKLOAD_SHA256,
|
||||
stages: STAGES.map(stagePlan), fullArms: 50, finalization: stagePlan(FINALIZE_STAGE),
|
||||
baselines: BASELINES, originalReceiptSha256: OLD_RECEIPT, resourceWatchdogMillis: RESOURCE_WATCHDOG_MILLIS,
|
||||
frozenH5: { revision: H5_REVISION, protocolSha256: H5_PROTOCOL_SHA256, comparatorSha256: H5_COMPARATOR_SHA256, inventoryHelperSha256: H5_INVENTORY_HELPER_SHA256, protocolDocumentSha256: H5_PROTOCOL_DOCUMENT_SHA256 },
|
||||
pendingPins: 'Future B2 has no descriptor until its actual new source and separately qualified native/application build are available. No H5 VM execution is released by this file.',
|
||||
note: 'Plan only; no SUT/build/VM/network or arm starts. Root and independent peer review plus qualification release required.' }, null, 2)); return;
|
||||
}
|
||||
const out = option('output-root') ?? '/home/ubuntu/native-ingestion/evidence/h5/browser';
|
||||
const context = await loadRelease(option('release'), stage, out);
|
||||
if (process.argv.includes('--validate-release') && !process.argv.includes('--execute')) {
|
||||
console.log(JSON.stringify({ valid: true, stage, requiredVariants: context.plan.requiredVariants,
|
||||
validationScope: 'Static immutable release/receipt/manifest/noise/stage-order bindings only; no live SUT/driver/build attestation and no arm execution', releaseFileSha256: context.releaseBytesSha256 })); return;
|
||||
}
|
||||
await checkFrozen(context); await mkdir(out, { recursive: true });
|
||||
// Keep the exact release bytes as a distinct artifact, so later stages never
|
||||
// require an owner to retain an overwriteable release-input filename.
|
||||
context.rootReleaseCopy = path.join(out, `${stage}-root-release.json`);
|
||||
const releaseBytes = await readFile(context.releaseFile);
|
||||
assert.equal(hash(releaseBytes), context.releaseBytesSha256, 'Root release changed before preservation');
|
||||
await writeFile(context.rootReleaseCopy, releaseBytes, { flag: 'wx' });
|
||||
await saveJson(path.join(out, `${stage}-release-record.json`), { ...context.release, releaseFile: context.releaseFile,
|
||||
releaseFileSha256: context.releaseBytesSha256, recordedAt: new Date().toISOString() }, true);
|
||||
if (context.plan.finalizationOnly) await finalizeSeries(context); else await browserSeries(context);
|
||||
}
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === selfFile) await main();
|
||||
@@ -0,0 +1,65 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import {pathToFileURL} from 'node:url';
|
||||
const directory=process.argv[2],harness=process.argv[3]??'/home/ubuntu/native-ingestion/harness-h5c';
|
||||
assert.ok(path.isAbsolute(directory)&&path.isAbsolute(harness));
|
||||
const hash=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const helper=path.join(harness,'tests/e2e/fixtures/native-ingestion-h5.mjs');
|
||||
assert.equal(hash(await readFile(helper)),'6d629f6eb3440aa089b35d683bbb20da894aafa9e4f29626ff199b0eaeffcb56');
|
||||
assert.equal(execFileSync('git',['rev-parse','HEAD'],{cwd:harness,encoding:'utf8'}).trim(),'e431ec63d41fe90c6fd68bd2fc2bb969feeca917');
|
||||
const {assertParticipantSetup,assertParticipantTopology,assertSettlementObservation,assertRecordedFailures,assessH5Arm,H5_REGISTRATION,H5_WORKLOADS,H5_LIMITS}=await import(pathToFileURL(helper).href);
|
||||
const evidenceBytes=await readFile(path.join(directory,'native-ingestion-evidence.json')),evidence=JSON.parse(evidenceBytes);
|
||||
const pwBytes=await readFile(path.join(directory,'playwright.json')),pw=JSON.parse(pwBytes),tests=[];
|
||||
assert.deepEqual(pw.errors??[],[]);
|
||||
function visit(value){if(Array.isArray(value))for(const child of value)visit(child);else if(value&&typeof value==='object'){
|
||||
if(Array.isArray(value.results)&&Object.hasOwn(value,'expectedStatus'))tests.push(value);
|
||||
for(const [key,child]of Object.entries(value))if(key!=='results')visit(child);
|
||||
}}
|
||||
visit(pw);assert.equal(tests.length,1);const test=tests[0];assert.equal(test.expectedStatus,'passed');assert.equal(test.results.length,1);
|
||||
const result=test.results[0];assert.equal(result.status,'passed');assert.equal(result.retry,0);assert.deepEqual(result.errors,[]);assert.ok(!result.error);
|
||||
const attachments=result.attachments.filter(value=>value.name==='native-ingestion-evidence');assert.equal(attachments.length,1);
|
||||
const attached=attachments[0].body?Buffer.from(attachments[0].body,'base64'):await readFile(path.resolve(directory,attachments[0].path));
|
||||
assert.equal(hash(attached),hash(evidenceBytes),'Owning Playwright attempt must attach these exact evidence bytes');
|
||||
const owner={reportFile:path.join(directory,'playwright.json'),reportSha256:hash(pwBytes),status:result.status,retry:result.retry,errors:result.errors,startedAt:result.startTime,durationMs:result.duration,timeoutMs:test.timeout};
|
||||
assert.equal(evidence.complete,true);assert.equal(evidence.observationComplete,true);assert.equal(evidence.protocolVersion,'v5c-independent-agents-complete-settlement-original-sla');
|
||||
assert.equal(evidence.measurementSha256,'2265aabe9c7344f5d4b9c8b08a5c6bbd5571a6e2de9ae7b72a8d1733250e609f');
|
||||
assert.equal(evidence.harnessProvenance.sourceRevision,'e431ec63d41fe90c6fd68bd2fc2bb969feeca917');assert.equal(evidence.harnessProvenance.sourceStatus,'');
|
||||
assert.equal(evidence.diagnosticOnly,false);assert.equal(evidence.browserRecording.trace,'off');assert.equal(evidence.browserRecording.video,'off');assert.equal(evidence.browserRecording.screencast,false);
|
||||
const outer=JSON.parse(await readFile(path.join(directory,'owner-receipt.json'),'utf8'));
|
||||
assert.equal(typeof outer.fullFixture,'boolean');assert.equal(typeof evidence.qualificationOnly,'boolean');
|
||||
assert.equal(outer.fullFixture,!evidence.qualificationOnly,'Owning command FULL/SMOKE mode must match its evidence');
|
||||
assert.equal(outer.sourceRevision,evidence.sourceRevision);assert.equal(outer.sourceRevision,evidence.compiledBuildRevision);assert.equal(evidence.serverArtifact,'compiled');
|
||||
assert.equal(outer.harnessRevision,evidence.harnessProvenance.sourceRevision);
|
||||
assert.equal(owner.timeoutMs,H5_LIMITS.fullInvocationMs);
|
||||
const expectedCases=outer.fullFixture?H5_WORKLOADS:[
|
||||
{name:'qualification',runs:3,events:16,intervalMs:25,prelude:0,stop:false},
|
||||
{name:'operator-stop',runs:1,events:512,intervalMs:100,prelude:0,stop:true},
|
||||
];
|
||||
const matrix=item=>Object.fromEntries(['name','runs','events','intervalMs','prelude','stop'].map(key=>[key,item[key]]));
|
||||
assert.deepEqual(evidence.cases.map(matrix),expectedCases.map(matrix),'Qualification requires the complete exact case matrix');
|
||||
assertParticipantSetup(evidence);const cases=[];
|
||||
for(const item of evidence.cases){
|
||||
assertParticipantTopology(evidence,item);
|
||||
const observed=assertSettlementObservation(item.h5.terminalObservation,item);
|
||||
const recorded=assertRecordedFailures(item.recordedFailures,item,evidence.companyId);
|
||||
cases.push({name:item.name,runs:item.runs,observedSettlement:observed,recordedFailures:recorded,
|
||||
observedTransitions:item.h5.terminalObservation.transitions,recordedFailureProof:item.recordedFailures,
|
||||
recordedRetryRate:recorded.uniqueEverFailedRuns/item.runs});
|
||||
}
|
||||
let assessment;
|
||||
if(!evidence.qualificationOnly){assessment=assessH5Arm(evidence,owner);assert.ok(assessment.misses.every(value=>H5_REGISTRATION.knownBaselineMisses.includes(value)));}
|
||||
assert.equal((await readFile(path.join(directory,'owner.exit'),'utf8')).trim(),'0');
|
||||
assert.equal(outer.passed,true);assert.equal(outer.cleanupComplete,true);
|
||||
const quiet=JSON.parse(await readFile(path.join(directory,'physical-quiet-after.json'),'utf8'));assert.equal(quiet.passed,true);assert.equal(quiet.platform,'linux');
|
||||
const port=JSON.parse(await readFile(path.join(directory,'port-after.json'),'utf8'));assert.equal(port.nodeBindAndListenPassed,true);
|
||||
const startup=await readFile(path.join(directory,'playwright.log'),'utf8');assert.match(startup,/Heartbeat\s+enabled\s*\(/,'Actual global scheduler must be enabled');
|
||||
const summary={complete:true,qualificationOnly:true,performanceAcceptanceEligible:false,
|
||||
scope:'Read-only post-run qualification validation with exact frozen H5c helpers; a full observation may retain original-SLA misses and is not an accepted A/A sample',
|
||||
evidenceSha256:hash(evidenceBytes),owningPlaywright:owner,helperSha256:hash(await readFile(helper)),
|
||||
actualGlobalSchedulerConfiguredEnabled:true,assessment,cases};
|
||||
await writeFile(path.join(directory,'independent-settlement-validation.json'),JSON.stringify(summary,null,2)+'\n',{flag:'wx'});
|
||||
console.log(JSON.stringify({complete:true,qualificationOnly:true,performanceAcceptanceEligible:false,assessment,
|
||||
cases:cases.map(value=>({name:value.name,runs:value.runs,recordedRetryRate:value.recordedRetryRate,recordedErrorClasses:value.recordedFailures.recordedErrorClasses}))}));
|
||||
@@ -0,0 +1,106 @@
|
||||
{
|
||||
"schema": "paperclip.native-ingestion.h5c-helper-sources.v1",
|
||||
"scope": "Exact reviewed task-helper source snapshots. Per-run owner receipts identify executed bytes; packaging executes no application, qualification or performance arm.",
|
||||
"frozenDriver": {
|
||||
"tag": "native-ingestion-h5c",
|
||||
"tagObject": "0ff37cd81dd922ce9def66fa656549e8e68d6a50",
|
||||
"commit": "e431ec63d41fe90c6fd68bd2fc2bb969feeca917",
|
||||
"lockSha256": "54a81b2ed3f1f85eaa085c034560ac248e384db34fbc7aba8c5a3846c29b3ce8",
|
||||
"measurementSha256": "2265aabe9c7344f5d4b9c8b08a5c6bbd5571a6e2de9ae7b72a8d1733250e609f",
|
||||
"comparatorSha256": "9e01acaae40251cb9730aa8a7e548d451b986112c20eae7ffc31759cf221393c",
|
||||
"inventoryHelperSha256": "76b897dd6b7a90e784e228d78d79633a6d7011fe3ea01737f3bef9c6113298e9",
|
||||
"protocolDocumentSha256": "5c78466770ecc3664339015d382ad6941109082976910a0f8ca2b88f78638c1f",
|
||||
"workloadSha256": "3eea2f24c0c0c38134492a0a17e1b9a398335c085dd248f03b14c14bb5a19835"
|
||||
},
|
||||
"recordedLayout": {
|
||||
"base": "/home/ubuntu/native-ingestion",
|
||||
"driver": "harness-h5c",
|
||||
"helperDirectory": "helpers",
|
||||
"appDirectories": {
|
||||
"B0": "aligned-baseline",
|
||||
"B1": "aligned-cursors",
|
||||
"B2": "aligned-batching"
|
||||
}
|
||||
},
|
||||
"recordedAppRevisions": {
|
||||
"B0": "bac53ec3bfeaf6546e7aaeaee516487b30534654",
|
||||
"B1": "07061a601cdc20603c8c7a46f9ddbad7ac5f5939",
|
||||
"B2": "ca7cba171cd5198ed569d75012edc858041cbc44"
|
||||
},
|
||||
"recordedQualifiedBuildReceiptSha256": "a1259ba760f3cbbe44817e333f9f620bcf3769fb0a6c16f9ce2f0f5b2ed60d94",
|
||||
"recordedInventorySummarySha256": "d3766fc4206d569d10fd02224433f1e9a2caa6992a7c62216e80af368aa4e44d",
|
||||
"rawB2Role": "Unoptimized batching base only. Final B2 needs its own immutable source, normal build and qualified inventory; none is supplied here.",
|
||||
"deliveryLockSha256AtPackaging": "7540075e96f9a93924a1e54775c17346c5760dd5d02902d83fb5228f918996ba",
|
||||
"deliveryLockWasExecutedDriver": false,
|
||||
"sourceOverlay": [
|
||||
{
|
||||
"file": "scripts/compare-native-ingestion-browser.ts",
|
||||
"sha256": "9e01acaae40251cb9730aa8a7e548d451b986112c20eae7ffc31759cf221393c",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "scripts/describe-native-ingestion-h5.mjs",
|
||||
"sha256": "e8c62f672f1fd67343abf3f05889a4ce4d1a926a07f391e9c4ef1d701f8c5eb3",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "scripts/native-ingestion-h5.test.mjs",
|
||||
"sha256": "b7dfee091a4b0a9c9af68e4b14cbb138c50fd07a49e77a6d08b122a550223921",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "scripts/native-ingestion-performance-gates.test.mjs",
|
||||
"sha256": "27782c2cd14d1ec7f680337ebe8ea4a9beebabe80915ae90fe01ea236a785abd",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/fixtures/native-ingestion-h5.md",
|
||||
"sha256": "5c78466770ecc3664339015d382ad6941109082976910a0f8ca2b88f78638c1f",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/fixtures/native-ingestion-h5.mjs",
|
||||
"sha256": "6d629f6eb3440aa089b35d683bbb20da894aafa9e4f29626ff199b0eaeffcb56",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
},
|
||||
{
|
||||
"file": "tests/e2e/native-event-ingestion.journey.ts",
|
||||
"sha256": "b49535ae6189a0891785f4c0ddbfe0c1c3490f610e752e60c912fea74852d53f",
|
||||
"byteIdenticalToFrozenDriver": true
|
||||
}
|
||||
],
|
||||
"files": [
|
||||
{
|
||||
"file": "native-ingestion-run-h5c-browser.mjs",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-run-h5c-browser.mjs",
|
||||
"bytes": 53940,
|
||||
"sha256": "6b6376b01de6cdc15ca05343a0bdd5c00424f1d6c546b9b4d61cd76716b253c4",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-linux-h5c-qualification.py",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-linux-h5c-qualification.py",
|
||||
"bytes": 6915,
|
||||
"sha256": "8349384d5a3f429536f80415238fd276e505cdbc2d9fb97798797188a3eb8f39",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-h5c-linux-readiness.mjs",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-h5c-linux-readiness.mjs",
|
||||
"bytes": 3446,
|
||||
"sha256": "57b336b01b0f36ffe9355993c04237f79a92a43c49b84fadb66630ccb18b54f4",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
},
|
||||
{
|
||||
"file": "native-ingestion-validate-h5c-evidence.mjs",
|
||||
"originalTaskFile": "/private/tmp/native-ingestion-validate-h5c-evidence.mjs",
|
||||
"bytes": 6438,
|
||||
"sha256": "dd69f0932472f548396644f72a9cad3862b758837652102aa840b852cb8c08ad",
|
||||
"byteIdenticalToReviewedOriginal": true,
|
||||
"packagingChanges": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
# H5b independent-agent observation registration (prospective; no execution release)
|
||||
# H5c canonical-settlement observation registration (prospective; no execution release)
|
||||
|
||||
H4c remains frozen at `7d7e7a88d562a95a1d407b76474763b666c181f5`
|
||||
(`5db4129bbdc1fa38775121aff5d0e68a8a11762519e41cfdf36eacd90ba7256b`).
|
||||
Its observed 120-second streaming failures remain failures. Aborted or profiled
|
||||
attempts cannot be reused as complete timing arms. H5b uses one normal agent per concurrent remote participant. It repairs the
|
||||
experiment topology while preserving observation of original deadline outcomes.
|
||||
attempts cannot be reused as complete timing arms. H5c keeps one normal agent per
|
||||
concurrent remote participant and waits for the product's canonical settlement,
|
||||
including supported same-run recovery, within the same terminal deadline.
|
||||
It does not optimize the product or turn earlier failures into passes.
|
||||
|
||||
The aligned fresh application baselines are B0 `bac53ec3bfeaf6546e7aaeaee516487b30534654`
|
||||
@@ -16,12 +17,14 @@ or matching binary is presumed before those artifacts exist. The historical
|
||||
a365/d8cd/914 application failures remain evidence of their recorded revisions.
|
||||
|
||||
Select H5 explicitly with
|
||||
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5b-observe-original-sla-independent-agents`.
|
||||
The new default H4d mode uses the same independent-agent geometry with the original
|
||||
120/180/900-second deadlines. Observation mode uses the explicit H5b protocol.
|
||||
`PAPERCLIP_INGESTION_OBSERVATION_MODE=h5c-observe-original-sla-independent-agents-settlement`.
|
||||
The new default H4e mode uses the same independent-agent geometry with the original
|
||||
120/180/900-second deadlines. Observation mode uses the explicit H5c protocol.
|
||||
Both modes hash the same nine source files, including the shared topology helper.
|
||||
Their source revision and measurement bytes must match for later confirmation.
|
||||
Old H4c/H5a protocols, geometry, arms and noise are ineligible for H5b.
|
||||
Old H4c/H5a protocols, geometry, arms and noise are ineligible for H5c. H5b179fe6
|
||||
and its H4d default proof use the premature terminal-status observer and are also
|
||||
ineligible. The new H4e proof must use the exact H5c source and geometry.
|
||||
|
||||
Run `node scripts/describe-native-ingestion-h5.mjs` to inspect the registration.
|
||||
This is a dry run only: it starts no app, provider, database or browser and
|
||||
@@ -77,7 +80,86 @@ A separate background presentation40P01 involved a run that succeeded. The
|
||||
retained log shows its coordinator→issue→run wait, but lacks PostgreSQL's full
|
||||
opposing lock cycle. Its cause and relationship to other contention are unproved.
|
||||
It is not evidence that the14 run failures came from a database deadlock. All
|
||||
historical failures remain separate; no failed H4c/H5a arm is reused as H5b noise.
|
||||
historical failures remain separate; no failed H4c/H5a arm is reused as H5c noise.
|
||||
|
||||
## Retained H5b failure and canonical settlement correction
|
||||
|
||||
H5b179fe6 on aligned B0bac53 admitted100 distinct normal agents before the common
|
||||
epoch. Its original120s poll failed with61/512 foreground markers; all512 became
|
||||
visible282.734s after the wait began. The observer stopped54.737s later on83
|
||||
succeeded and17 failed statuses. Independent PostgreSQL checks found all51,200
|
||||
lossless deltas and one accepted result per run. Qualification failed; history
|
||||
and Stop were not reached. This arm is not noise or performance evidence.
|
||||
|
||||
All17 errors were provider_transport_failed with the generic message that the
|
||||
runner did not durably suspend before checkpoint. Existing run logs instead
|
||||
show runnerSuspended=true and providerDrained=false: the exact runner.drain
|
||||
receipt was still pending after its five-second drain window. Why the receipt
|
||||
was late remains unproved. These were not the earlier shared-agent lock errors.
|
||||
All17 finalizers were retryable_failure at attempt1 with a scheduled+30s workspace
|
||||
retry; application teardown finished before the earliest retry was due. ACTIVE
|
||||
leases and prepared workspace-sync receipts remained unfinished. Later recovery
|
||||
was not observed and is not presumed to pass.
|
||||
|
||||
The exact B0bac53 run-failure-report.ts44–50 contract states:
|
||||
|
||||
> A native run that waits for a same-run retry is stored as `failed` with
|
||||
> `nativePhase: "retryable_failure"`. That is not terminal: the retry or the
|
||||
> workspace copy-back retry can still finish the run as succeeded.
|
||||
|
||||
Normal company-run lists omit
|
||||
runtimeMode/nativePhase. Normal GET /api/heartbeat-runs/:id exposes both through
|
||||
the existing full run columns. Inside the same single180_000ms terminal poll,
|
||||
resolve only failed list candidates through that detail GET and classify its
|
||||
current result. Keep failed/native/retryable_failure pending; permanent/unknown
|
||||
failures and GET errors reject. The observer sends no retry or provider command.
|
||||
|
||||
Record bounded state changes per run with time, status, runtimeMode, nativePhase
|
||||
and errorCode. Missing list-only fields are null, not inferred. Cap at16 changes
|
||||
per run and fail on overflow rather than discard observations. Retain total
|
||||
unique retryable runs, recovered runs, nonretryable failures and per-error-class
|
||||
unique-run counts. Every observed retry must recover successfully within
|
||||
the unchanged180s window; a deadline miss remains censored failure, never a
|
||||
completed latency. All100 successful native runs, exact deltas/results, reload,
|
||||
lease cleanup, existing-history and local Stop gates remain required.
|
||||
|
||||
After recorded settlement, acknowledgement, lease timing, reload and normal
|
||||
event verification, take one company-and-exact-run-set READ ONLY REPEATABLE READ
|
||||
snapshot through the existing owned PostgreSQL helper. Retain current error-code
|
||||
fields and the closed schema/errorCode/observedAt fields of
|
||||
result_json.recoveredExecutionFailure, plus descriptive finalization attempt.
|
||||
Require every actual run, exact identities, settled status, valid shapes and a
|
||||
closed connection before the next case. No event HTTP or database polling loop
|
||||
is added. The existing independent completion upper bound includes this check;
|
||||
the measured settlement and lease timestamps stay unchanged.
|
||||
|
||||
The durable recovery marker is copied from the run's error fields when successful
|
||||
finalization clears them (native-run-projection.ts114–121 in B0bac53). It can
|
||||
therefore retain a failure recovered before terminal-window polling. Union its
|
||||
codes and current recorded errors with observed transition codes by run identity.
|
||||
The normal Stop path's closed cancelled code is retained as cancellation
|
||||
evidence (run-control.ts1345); other current Stop error codes remain failures.
|
||||
Use these all-recorded unique-ever-failed-run counts/rates for comparison, and
|
||||
retain the observed-only counters separately. This marker records the retained
|
||||
failure, not an exhaustive sequence of every error class or attempt. Do not claim
|
||||
omniscient error history. Finalization attempt is a claim counter that can be
|
||||
preserved across workspace repair (native-run-finalizer.ts237–240); it is not a
|
||||
failure count and supplies no rate gate.
|
||||
|
||||
Successful committed ordinary runs must have both current error fields cleared,
|
||||
as required by native-run-projection.ts180–193. A still-current error fails the
|
||||
proof; a succeeded status alone cannot turn it into recovered failure evidence.
|
||||
|
||||
Fresh A/A retains all histories and freezes, per case, total and per-error-class
|
||||
all-recorded unique-ever-failed-run rates plus the maximum absolute paired rate difference. Zero baseline
|
||||
counts are valid; no relative division is used. Each candidate may use only
|
||||
error classes observed in that baseline A/A. Every paired total/per-class rate
|
||||
increase and its median must stay within that frozen absolute noise envelope.
|
||||
A single-arm retry storm cannot hide behind the median. Recovery and its final
|
||||
backlog remain inside drainage timing; observing recovery does not mean the
|
||||
earlier errors disappeared. All recorded failures must have recovered, and every
|
||||
run still needs its exact independently persisted accepted result. No product
|
||||
retry policy or drain limit is changed.
|
||||
|
||||
## Fixed work and observation boundaries
|
||||
|
||||
@@ -95,7 +177,7 @@ is added. Fake-plugin streaming remains separate from real-cloud Stop proof.
|
||||
| Boundary | Original requirement | H5 observation |
|
||||
| --- | --- | --- |
|
||||
| Foreground markers | Original Playwright120s wait | Continue to an absolute600s ceiling from that same wait start |
|
||||
| All terminal runs |180s after foreground completion (and Stop click) | Same180s after eventual foreground completion; hard failure at the ceiling |
|
||||
| Canonically settled runs |180s after foreground completion (and Stop click) | Same single180s after eventual foreground completion; supported retries remain pending, permanent failures reject, deadline misses fail |
|
||||
| Stop acknowledgement and leases | Existing30s polls and ordering | Unchanged; full click→acknowledgement metric retained |
|
||||
| Invocation | Original900s test budget | Actual1200s observation timeout, separately disclosed |
|
||||
|
||||
@@ -131,7 +213,7 @@ status, errors and configured timeout. Its observed total duration is an
|
||||
independent conservative assessment of the original900s budget; hooks/teardown
|
||||
can be treated differently by the original timeout. It is not an exact original
|
||||
timer-expiration observation. A final candidate must have total duration<900s
|
||||
in every arm and later pass one unchanged H4d default-mode full invocation on the
|
||||
in every arm and later pass one unchanged H4e default-mode full invocation on the
|
||||
same final SUT revision and full qualified artifact inventory.
|
||||
|
||||
## Complete evidence and eligibility
|
||||
@@ -167,7 +249,7 @@ not itself attest that the live filesystem stayed unchanged.
|
||||
| --- | --- | --- |
|
||||
| Fresh B0/B1 A/A arms | Known burst512 streaming120 miss; explicit conservative observed-duration900 miss | Complete finite correct observation; root-reviewed noise suitability |
|
||||
| B0→B1 cursor-only candidate | Only an exact registered known miss also present in the paired baseline: burst512 streaming120 or conservative observed-duration900 | No additional deadline failures; latency/control nonregression plus separate incremental SQL evidence; never claim SLA fixed |
|
||||
| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later H4d proof |
|
||||
| Final B2 candidate in either comparison | None | Every arm meets original streaming/control checks and conservative900 gate; loaded improvements and later H4e proof |
|
||||
|
||||
Only burst512's streaming miss is registered as a known workload SLA failure.
|
||||
New failures in other workloads require diagnosis. The existing Stop poll
|
||||
@@ -190,13 +272,13 @@ improve; other fixed latency/control metrics must not regress outside noise.
|
||||
Do not infer p99 from sparse observations. There is no new fixed percentage A/A
|
||||
cutoff: root must reject unsuitable noisy/nonstationary calibration explicitly.
|
||||
|
||||
Use `--observation-mode=h5b-observe-original-sla-independent-agents` for the comparator. Candidate
|
||||
Use `--observation-mode=h5c-observe-original-sla-independent-agents-settlement` for the comparator. Candidate
|
||||
comparisons also require `--candidate-policy=cursor-nonregression --expect=nonregression`
|
||||
or `--candidate-policy=final-b2`. Final comparisons expose `performancePassed`
|
||||
separately; `acceptancePassed` remains false until `--original-h4d-proof` and
|
||||
`--original-h4d-inventory` establish the later fresh H4d default-deadline functional pass.
|
||||
That H4d proof is full, unprofiled, retry0, default mode with original120/180/900s
|
||||
bounds, on the same frozen H5b harness source, nine-file measurement bytes and
|
||||
separately; `acceptancePassed` remains false until `--original-h4e-proof` and
|
||||
`--original-h4e-inventory` establish the later fresh H4e default-deadline functional pass.
|
||||
That H4e proof is full, unprofiled, retry0, default mode with original120/180/900s
|
||||
bounds, on the same frozen H5c harness source, nine-file measurement bytes and
|
||||
independent-agent geometry, and the exact final SUT revision/qualified inventory.
|
||||
It uses a new disposable company and starts after all candidate arms. Old7d/H4c
|
||||
proofs and H5a observations cannot confirm this geometry. Prior failed H4c attempts remain in the record.
|
||||
@@ -206,7 +288,7 @@ ordering; no absolute click-to-acknowledgement deadline is added.
|
||||
Use the explicit H5 `--require-performance` gate for the two final comparison
|
||||
stages while that later proof is pending; this requires every performance and
|
||||
candidate original-SLA gate, but grants no final acceptance. After the one later
|
||||
H4d default-mode qualification, recompute both reports against their unchanged paired
|
||||
H4e default-mode qualification, recompute both reports against their unchanged paired
|
||||
manifests with the proof and `--require-acceptance`. This re-evaluates retained
|
||||
evidence; it neither reuses an arm in a different comparison nor reruns failures.
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const H5_MODE = 'h5b-observe-original-sla-independent-agents';
|
||||
export const H5_PROTOCOL = 'v5b-independent-agents-complete-observation-original-sla';
|
||||
export const H4D_PROTOCOL = 'v4d-independent-agents-original-deadlines';
|
||||
export const H5_MODE = 'h5c-observe-original-sla-independent-agents-settlement';
|
||||
export const H5_PROTOCOL = 'v5c-independent-agents-complete-settlement-original-sla';
|
||||
export const H4E_PROTOCOL = 'v4e-independent-agents-settlement-original-deadlines';
|
||||
export const H5_TOPOLOGY = Object.freeze({ mode: 'one-normal-agent-per-concurrent-participant',
|
||||
fullRemoteAgentPoolSize: 100, agentSetupConcurrency: 8, maxConcurrentRuns: 50,
|
||||
poolReuse: 'only-across-sequential-cases' });
|
||||
@@ -12,6 +12,7 @@ export const H5_BASELINE_REVISIONS = Object.freeze({ B0: 'bac53ec3bfeaf6546e7aae
|
||||
export const ORIGINAL_BUDGETS = Object.freeze({ providerReadinessMs: 300_000, fullInvocationMs: 900_000,
|
||||
streamingMs: 120_000, terminalDrainageMs: 180_000, stopAcknowledgementMs: 30_000, remoteLeaseCleanupMs: 30_000 });
|
||||
export const H5_LIMITS = Object.freeze({ streamingMs: 600_000, terminalDrainageMs: 180_000, fullInvocationMs: 1_200_000 });
|
||||
export const SETTLEMENT_MAX_TRANSITIONS_PER_RUN = 16;
|
||||
export const H5_WORKLOADS = Object.freeze([
|
||||
{ name: 'paced', runs: 1, events: 128, intervalMs: 25, prelude: 0, stop: false },
|
||||
{ name: 'burst16', runs: 1, events: 16, intervalMs: 0, prelude: 0, stop: false },
|
||||
@@ -22,14 +23,19 @@ export const H5_WORKLOADS = Object.freeze([
|
||||
]);
|
||||
export const H5_REGISTRATION = Object.freeze({ mode: H5_MODE, protocolVersion: H5_PROTOCOL,
|
||||
baselineRevisions: H5_BASELINE_REVISIONS,
|
||||
topology: H5_TOPOLOGY, originalDeadlineProtocol: H4D_PROTOCOL,
|
||||
topology: H5_TOPOLOGY, originalDeadlineProtocol: H4E_PROTOCOL,
|
||||
settlement: { classification: 'Failed native retryable_failure remains pending within the original terminal window',
|
||||
detailReads: 'Only failed run-list candidates use the normal detail GET; no event polling or retry dispatch',
|
||||
maxTransitionsPerRun: SETTLEMENT_MAX_TRANSITIONS_PER_RUN,
|
||||
retryRatePolicy: 'No new recorded error class after frozen A/A; every paired and median total/per-class unique-ever-failed-run rate increase must stay within absolute paired A/A noise',
|
||||
recordedFailures: 'Post-timing exact-run-set read-only snapshot unions current and durable recovered error codes with observed transitions; retained recovery metadata is last-failure evidence, not exhaustive per-class history' },
|
||||
originalBudgets: ORIGINAL_BUDGETS, observationLimits: H5_LIMITS, workloads: H5_WORKLOADS,
|
||||
pairs: 5, order: 'baseline-first on even pair index; candidate-first on odd pair index',
|
||||
stages: ['B0 A/A', 'B1 A/A', 'B0→B1 nonregression', 'B1→B2 improvement', 'B0→B2 improvement'],
|
||||
knownBaselineMisses: ['burst512:streaming120', 'invocation:observed-total-duration900'],
|
||||
baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of the H4d original deadlines',
|
||||
baselineEligibility: 'Finite complete correct observation; SLA misses remain failures of the H4e original deadlines',
|
||||
cursorCandidateEligibility: 'Nonregression; only the paired baseline exact registered known misses may be inherited',
|
||||
finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later H4d default-mode functional PASS on the same harness source and independent-agent geometry required',
|
||||
finalCandidateEligibility: 'Every original SLA passes in every arm; independently assessed total duration <900s; later H4e default-mode functional PASS on the same harness source and independent-agent geometry required',
|
||||
noise: 'Fresh B0 and B1 A/A frozen before any further product implementation; no old abort/profile samples',
|
||||
performance: 'Loaded p95 emission/intended-to-visible and terminal drainage improve beyond frozen noise in median and at least4/5 pairs; all other metrics nonregress',
|
||||
});
|
||||
@@ -156,11 +162,178 @@ export async function observeStreaming({ readCount, poll, target, now = Date.now
|
||||
|
||||
export function originalTerminal(streaming, terminal) {
|
||||
return streaming.original.status === 'passed'
|
||||
? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null : 'original_terminal_deadline',
|
||||
? { status: terminal.complete ? 'passed' : 'failed', reason: terminal.complete ? null
|
||||
: terminal.censored === false && terminal.error ? 'original_terminal_failure' : 'original_terminal_deadline',
|
||||
startedAtMs: terminal.startedAtMs, deadlineMs: terminal.deadlineMs, assessedAtMs: terminal.assessedAtMs }
|
||||
: { status: 'not-reached', reason: 'original_streaming_deadline', startedAtMs: null, deadlineMs: null, assessedAtMs: null };
|
||||
}
|
||||
|
||||
/** The run list omits nativePhase. Resolve failed rows through the normal run
|
||||
* detail API before deciding whether the existing terminal window has drained. */
|
||||
export async function observeRunSettlement({ readRuns, readRun, poll, expectedRuns, now = Date.now, onOutcome = _value => {} }) {
|
||||
const startedAtMs = now();
|
||||
const outcome = { startedAtMs, deadlineMs: startedAtMs + ORIGINAL_BUDGETS.terminalDrainageMs,
|
||||
assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0, pendingRetryRuns: 0,
|
||||
lastStatuses: [], transitions: [], uniqueRetryableRuns: 0, recoveredRuns: 0, nonretryableFailures: 0,
|
||||
retryableErrorClasses: {}, complete: false, censored: false };
|
||||
const previous = new Map();
|
||||
let runs = [];
|
||||
onOutcome(outcome);
|
||||
try {
|
||||
await poll(async () => {
|
||||
runs = [];
|
||||
for (const listed of await readRuns()) {
|
||||
const run = listed.status === 'failed' ? await readRun(listed.id) : listed;
|
||||
assert.equal(run.id, listed.id, 'Run detail belongs to a different run');
|
||||
if (listed.status === 'failed') {
|
||||
assert.ok(typeof run.runtimeMode === 'string', 'Run detail must expose runtimeMode');
|
||||
if (run.runtimeMode === 'native') assert.ok(Object.hasOwn(run, 'nativePhase'), 'Run detail must expose nativePhase');
|
||||
}
|
||||
runs.push(run);
|
||||
}
|
||||
assert.ok(runs.length <= expectedRuns && new Set(runs.map(run => run.id)).size === runs.length,
|
||||
'Terminal observation contains duplicate or additional runs');
|
||||
const observedAtMs = now();
|
||||
const statuses = runs.map(run => ({ runId: run.id, status: run.status,
|
||||
runtimeMode: run.runtimeMode ?? null, nativePhase: run.nativePhase ?? null, errorCode: run.errorCode ?? null }));
|
||||
for (const state of statuses) {
|
||||
const signature = JSON.stringify(state);
|
||||
if (previous.get(state.runId) !== signature) {
|
||||
assert.ok(outcome.transitions.filter(row => row.runId === state.runId).length < SETTLEMENT_MAX_TRANSITIONS_PER_RUN,
|
||||
'Terminal state transition evidence overflow; no observation may be dropped');
|
||||
outcome.transitions.push({ observedAtMs, ...state });
|
||||
}
|
||||
previous.set(state.runId, signature);
|
||||
}
|
||||
const pending = run => run.status === 'failed' && run.runtimeMode === 'native' && run.nativePhase === 'retryable_failure';
|
||||
const settled = run => ['succeeded', 'cancelled', 'failed'].includes(run.status) && !pending(run);
|
||||
Object.assign(outcome, { lastObservedAtMs: observedAtMs, observedRuns: runs.length,
|
||||
terminalRuns: runs.filter(settled).length, pendingRetryRuns: runs.filter(pending).length, lastStatuses: statuses,
|
||||
...summarizeSettlementTransitions(outcome.transitions) });
|
||||
onOutcome(outcome);
|
||||
assert.ok(!runs.some(run => run.status === 'timed_out' || run.status === 'failed' && !pending(run)),
|
||||
'Terminal run failed; automatic retry is not pending');
|
||||
return runs.length === expectedRuns && runs.every(settled);
|
||||
}, ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
assert.equal(outcome.recoveredRuns, outcome.uniqueRetryableRuns, 'Every observed retryable run must recover successfully');
|
||||
outcome.assessedAtMs = now(); outcome.complete = true; onOutcome(outcome);
|
||||
return { runs, outcome };
|
||||
} catch (error) {
|
||||
outcome.assessedAtMs = now();
|
||||
outcome.censored = error?.matcherResult?.name === 'toBe' && error.matcherResult.pass === false
|
||||
&& error.message.includes(`Timeout ${ORIGINAL_BUDGETS.terminalDrainageMs}ms exceeded while waiting on the predicate`);
|
||||
outcome.error = String(error); onOutcome(outcome); throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function summarizeSettlementTransitions(transitions) {
|
||||
const retryable = new Set(), recovered = new Set(), permanent = new Set(), classes = new Map();
|
||||
for (const row of transitions) {
|
||||
if (row.status === 'failed' && row.runtimeMode === 'native' && row.nativePhase === 'retryable_failure') {
|
||||
assert.ok(typeof row.errorCode === 'string' && row.errorCode.length > 0, 'Retryable failure needs its observed error class');
|
||||
retryable.add(row.runId);
|
||||
const runs = classes.get(row.errorCode) ?? new Set(); runs.add(row.runId); classes.set(row.errorCode, runs);
|
||||
} else if (row.status === 'failed' || row.status === 'timed_out') permanent.add(row.runId);
|
||||
if (row.status === 'succeeded' && retryable.has(row.runId)) recovered.add(row.runId);
|
||||
}
|
||||
return { uniqueRetryableRuns: retryable.size, recoveredRuns: recovered.size, nonretryableFailures: permanent.size,
|
||||
retryableErrorClasses: Object.fromEntries([...classes].sort(([a], [b]) => a.localeCompare(b)).map(([code, runs]) => [code, runs.size])) };
|
||||
}
|
||||
|
||||
export function assertSettlementObservation(value, item) {
|
||||
assert.ok(value, 'Missing canonical run settlement observation');
|
||||
assert.equal(value.complete, true); assert.equal(value.censored, false);
|
||||
assert.ok(Number.isSafeInteger(value.startedAtMs) && value.startedAtMs > 0);
|
||||
assert.equal(value.deadlineMs, value.startedAtMs + ORIGINAL_BUDGETS.terminalDrainageMs);
|
||||
assert.equal(value.assessedAtMs, item.settledAtMs);
|
||||
assert.ok(value.lastObservedAtMs >= value.startedAtMs && value.lastObservedAtMs <= value.assessedAtMs
|
||||
&& value.assessedAtMs <= value.deadlineMs, 'Settlement must fit the original terminal window');
|
||||
assert.equal(value.observedRuns, item.runs); assert.equal(value.terminalRuns, item.runs); assert.equal(value.pendingRetryRuns, 0);
|
||||
assert.deepEqual(value.lastStatuses.map(row => row.runId).sort(), [...item.runIds].sort());
|
||||
assert.ok(value.lastStatuses.every(row => row.status === (item.stop ? 'cancelled' : 'succeeded')), 'Failed runs are not settled success');
|
||||
assert.ok(Array.isArray(value.transitions) && value.transitions.length >= item.runs
|
||||
&& value.transitions.length <= item.runs * SETTLEMENT_MAX_TRANSITIONS_PER_RUN, 'Missing or overflowing bounded state history');
|
||||
const last = new Map(), counts = new Map(); let priorTime = value.startedAtMs;
|
||||
for (const row of value.transitions) {
|
||||
assert.ok(item.runIds.includes(row.runId), 'State history belongs to another run');
|
||||
assert.ok(Number.isSafeInteger(row.observedAtMs) && row.observedAtMs >= priorTime
|
||||
&& row.observedAtMs <= value.lastObservedAtMs, 'Invalid state history timestamp');
|
||||
priorTime = row.observedAtMs;
|
||||
for (const field of ['runtimeMode', 'nativePhase', 'errorCode']) assert.ok(row[field] === null || typeof row[field] === 'string');
|
||||
const count = (counts.get(row.runId) ?? 0) + 1; counts.set(row.runId, count);
|
||||
assert.ok(count <= SETTLEMENT_MAX_TRANSITIONS_PER_RUN, 'State history overflow');
|
||||
last.set(row.runId, row);
|
||||
}
|
||||
for (const row of value.lastStatuses) {
|
||||
const { observedAtMs: _at, ...state } = last.get(row.runId) ?? {};
|
||||
assert.deepEqual(state, row, 'Final state must match retained state history');
|
||||
}
|
||||
const summary = summarizeSettlementTransitions(value.transitions);
|
||||
for (const [field, expected] of Object.entries(summary)) assert.deepEqual(value[field], expected, 'Retry ledger must match retained transitions');
|
||||
assert.equal(summary.nonretryableFailures, 0, 'Nonretryable failures cannot qualify');
|
||||
assert.equal(summary.recoveredRuns, summary.uniqueRetryableRuns, 'Every observed retryable run must recover successfully');
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function assertRecordedFailures(proof, item, companyId) {
|
||||
const observed = assertSettlementObservation(item.h5?.terminalObservation ?? item.terminalObservation, item);
|
||||
assert.equal(proof?.kind, 'recorded-run-failure-codes', 'Missing post-timing recorded failure proof');
|
||||
assert.equal(proof.transaction, 'READ ONLY REPEATABLE READ'); assert.equal(proof.observedReadOnly, 'on');
|
||||
assert.equal(proof.observedIsolation, 'repeatable read');
|
||||
const after = Math.max(item.settledAtMs, item.remoteLeasesCheckedAtMs ?? 0, item.stopAcknowledgedAtMs ?? 0,
|
||||
item.localStopProof?.checkedAtMs ?? 0);
|
||||
assert.ok(Number.isSafeInteger(proof.queryStartedAtMs) && proof.queryStartedAtMs >= after
|
||||
&& Number.isSafeInteger(proof.checkedAtMs) && proof.checkedAtMs >= proof.queryStartedAtMs
|
||||
&& Number.isSafeInteger(proof.connectionClosedAtMs) && proof.connectionClosedAtMs >= proof.checkedAtMs,
|
||||
'Close the post-timing failure snapshot after settlement and lease timing');
|
||||
assert.equal(proof.rows?.length, item.runs, 'Recorded failure proof needs every actual run');
|
||||
assert.deepEqual(proof.rows.map(row => row.run_id).sort(), [...item.runIds].sort(), 'Recorded failure run set mismatch');
|
||||
assert.equal(new Set(proof.rows.map(row => row.run_id)).size, item.runs);
|
||||
const classes = new Map(), ever = new Set(), recovered = new Set();
|
||||
const add = (runId, code) => {
|
||||
assert.ok(typeof code === 'string' && code.length > 0 && code.length <= 128, 'Missing closed recorded error code');
|
||||
ever.add(runId); const ids = classes.get(code) ?? new Set(); ids.add(runId); classes.set(code, ids);
|
||||
};
|
||||
const transitions = (item.h5?.terminalObservation ?? item.terminalObservation).transitions;
|
||||
for (const row of transitions) if (row.status === 'failed') add(row.runId, row.errorCode);
|
||||
for (const row of proof.rows) {
|
||||
assert.equal(row.company_id, companyId, 'Recorded failure company mismatch');
|
||||
assert.equal(row.agent_id, item.controllerAuthority.rows.find(authority => authority.run_id === row.run_id)?.agent_id,
|
||||
'Recorded failure agent mismatch');
|
||||
assert.equal(row.runtime_mode, 'native');
|
||||
assert.equal(row.status, item.stop ? 'cancelled' : 'succeeded', 'Recorded run is not successfully settled');
|
||||
if (!item.stop) assert.equal(row.native_phase, 'committed', 'Recorded native finalization is not committed');
|
||||
else assert.ok(row.native_phase === null || typeof row.native_phase === 'string', 'Invalid recorded cancellation phase');
|
||||
assert.ok(typeof row.current_has_error === 'boolean');
|
||||
assert.ok(row.current_error_code === null || typeof row.current_error_code === 'string');
|
||||
if (!item.stop) {
|
||||
assert.equal(row.current_has_error, false, 'Successful committed runs must clear their current execution error');
|
||||
assert.equal(row.current_error_code, null, 'Successful committed runs must clear their current execution error code');
|
||||
}
|
||||
// The normal Stop path writes the closed code "cancelled". Retain it as
|
||||
// cancellation evidence; every other recorded code remains failure evidence.
|
||||
if ((row.current_has_error || row.current_error_code !== null) && !(item.stop && row.current_error_code === 'cancelled'))
|
||||
add(row.run_id, row.current_error_code);
|
||||
if (row.recovered_type === null || row.recovered_type === 'null') {
|
||||
for (const key of ['recovered_schema', 'recovered_error_code', 'recovered_observed_at']) assert.equal(row[key], null);
|
||||
} else {
|
||||
assert.equal(row.recovered_type, 'object'); assert.equal(row.recovered_schema, 'paperclip.recovered_execution_failure.v1');
|
||||
assert.ok(typeof row.recovered_observed_at === 'string' && Number.isFinite(Date.parse(row.recovered_observed_at))
|
||||
&& Date.parse(row.recovered_observed_at) <= proof.checkedAtMs, 'Invalid recovered failure observation time');
|
||||
add(row.run_id, row.recovered_error_code);
|
||||
}
|
||||
assert.ok(row.finalization_attempt === null || Number.isSafeInteger(row.finalization_attempt) && row.finalization_attempt >= 0,
|
||||
'Invalid descriptive finalization attempt');
|
||||
if (ever.has(row.run_id) && row.status === 'succeeded') recovered.add(row.run_id);
|
||||
}
|
||||
assert.equal(recovered.size, ever.size, 'Every recorded failed run must have recovered successfully');
|
||||
assert.ok(item.durable.every(row => row.resultAcceptedEvents === (item.stop ? 0 : 1)), 'Every run needs its exact independently persisted result');
|
||||
if (item.h5?.independentDrain) assert.ok(item.h5.independentDrain.verifiedAtMs >= proof.connectionClosedAtMs,
|
||||
'Independent drainage must include the closed recorded-failure snapshot');
|
||||
return { ...observed, uniqueEverFailedRuns: ever.size, allRecordedRecoveredRuns: recovered.size,
|
||||
recordedErrorClasses: Object.fromEntries([...classes].sort(([a], [b]) => a.localeCompare(b)).map(([code, ids]) => [code, ids.size])) };
|
||||
}
|
||||
|
||||
export function assessH5Arm(report, owner) {
|
||||
assert.equal(report.protocolVersion, H5_PROTOCOL);
|
||||
assert.equal(report.observationMode, H5_MODE);
|
||||
@@ -209,6 +382,7 @@ export function assessH5Arm(report, owner) {
|
||||
assert.equal(streaming.error, null);
|
||||
assert.equal(value.terminalOriginal.status, 'passed');
|
||||
}
|
||||
assertSettlementObservation(value.terminalObservation, item);
|
||||
assert.equal(value.terminalObservation.complete, true, 'H5 terminal observation must drain');
|
||||
assert.equal(value.terminalObservation.censored, false);
|
||||
assert.equal(value.terminalObservation.deadlineMs, value.terminalObservation.startedAtMs + H5_LIMITS.terminalDrainageMs);
|
||||
|
||||
@@ -7,8 +7,8 @@ import { hostname } from "node:os";
|
||||
import path from "node:path";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { json } from "./agent-chat.shared";
|
||||
import { h5Enabled, H5_MODE, H5_PROTOCOL, H4D_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS,
|
||||
observeStreaming, originalTerminal, prepareParticipantAgents, assertParticipantTopology } from "./fixtures/native-ingestion-h5.mjs";
|
||||
import { h5Enabled, H5_MODE, H5_PROTOCOL, H4E_PROTOCOL, H5_TOPOLOGY, H5_LIMITS, H5_WORKLOAD_SHA256, ORIGINAL_BUDGETS,
|
||||
observeStreaming, observeRunSettlement, originalTerminal, prepareParticipantAgents, assertParticipantTopology, assertRecordedFailures } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
type SeenMarker = { markerCode: string; index: number; visibleMs: number };
|
||||
type Emission = { nonce: string; markerCode: string; index: number; intendedMs: number; emittedMs: number; receivedAtMs: number };
|
||||
@@ -44,7 +44,7 @@ const stats = (values: number[]) => {
|
||||
|
||||
async function runtimeAuthority(input: {
|
||||
appRoot: string; companyId: string; environmentId: string; issueIds: string[]; transport: "local" | "sandbox-remote";
|
||||
afterStop?: boolean;
|
||||
afterStop?: boolean; recordedFailureRunIds?: string[];
|
||||
}) {
|
||||
const configuration = process.env.PAPERCLIP_E2E_SERVER_CONFIG;
|
||||
if (!configuration) throw new Error("Missing owned server configuration");
|
||||
@@ -68,6 +68,21 @@ async function runtimeAuthority(input: {
|
||||
current_setting('transaction_isolation') as isolation`;
|
||||
expect(settings[0].read_only).toBe("on");
|
||||
expect(settings[0].isolation).toBe("repeatable read");
|
||||
if (input.recordedFailureRunIds) {
|
||||
const queryStartedAtMs = Date.now();
|
||||
const rows = await tx`select r.id as run_id, r.company_id, r.agent_id, r.runtime_mode, r.status, r.native_phase,
|
||||
r.error_code as current_error_code, r.error is not null as current_has_error,
|
||||
jsonb_typeof(r.result_json->'recoveredExecutionFailure') as recovered_type,
|
||||
r.result_json->'recoveredExecutionFailure'->>'schema' as recovered_schema,
|
||||
r.result_json->'recoveredExecutionFailure'->>'errorCode' as recovered_error_code,
|
||||
r.result_json->'recoveredExecutionFailure'->>'observedAt' as recovered_observed_at,
|
||||
f.attempt as finalization_attempt
|
||||
from heartbeat_runs r left join native_run_finalizations f on f.run_id = r.id and f.company_id = r.company_id
|
||||
where r.company_id = ${input.companyId} and r.id = any(${input.recordedFailureRunIds}::uuid[])`;
|
||||
return { kind: "recorded-run-failure-codes", queryStartedAtMs, checkedAtMs: Date.now(),
|
||||
transaction: "READ ONLY REPEATABLE READ", observedReadOnly: settings[0].read_only,
|
||||
observedIsolation: settings[0].isolation, rows };
|
||||
}
|
||||
const rows = await tx`select r.id as run_id, r.company_id, r.agent_id, r.native_issue_id as issue_id, r.runtime_mode,
|
||||
r.native_session_id, r.runner_instance_id, r.status, r.process_pid, r.process_group_id, r.process_started_at,
|
||||
r.runner_profile_json->'nativeExecutionInput'->'binding'->>'executionWorkspaceId' as admitted_execution_workspace_id,
|
||||
@@ -115,7 +130,7 @@ async function runtimeAuthority(input: {
|
||||
} finally { await sql.end({ timeout: 5 }); }
|
||||
const connectionClosedAtMs = Date.now();
|
||||
let localJournal;
|
||||
if (input.transport === "local" && !input.afterStop) {
|
||||
if (input.transport === "local" && !input.afterStop && !input.recordedFailureRunIds) {
|
||||
expect(input.issueIds).toHaveLength(1);
|
||||
const base = await realpath(path.join(path.dirname(configuration), "runtime/paperclip-runner/durable-sessions"));
|
||||
expect(base.startsWith(`${ownedHome}${path.sep}`)).toBe(true);
|
||||
@@ -275,7 +290,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
const browserRecording = { trace: captureMode(info.project.use.trace), video: captureMode(info.project.use.video),
|
||||
screencast: captureMode(info.project.use.trace) !== "off" || captureMode(info.project.use.video) !== "off" };
|
||||
const report: Record<string, unknown> = {
|
||||
protocolVersion: h5 ? H5_PROTOCOL : H4D_PROTOCOL, transport, bodyStartedAtMs,
|
||||
protocolVersion: h5 ? H5_PROTOCOL : H4E_PROTOCOL, transport, bodyStartedAtMs,
|
||||
agentTopology: { ...H5_TOPOLOGY, remoteAgentPoolSize: Math.max(...cases.map(item => item.runs)) },
|
||||
...(h5 ? { observationMode: H5_MODE, workloadSha256: H5_WORKLOAD_SHA256, observationLimits: H5_LIMITS,
|
||||
observationComplete: false, actualPlaywrightTimeoutMs: info.timeout,
|
||||
@@ -555,25 +570,20 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
}
|
||||
active.phase = "terminal-drainage";
|
||||
let runs: any[] = [];
|
||||
const terminalStartedAtMs = h5 ? Date.now() : undefined;
|
||||
const terminalObservation = h5 ? { startedAtMs: terminalStartedAtMs!, deadlineMs: terminalStartedAtMs! + H5_LIMITS.terminalDrainageMs,
|
||||
assessedAtMs: 0, lastObservedAtMs: 0, observedRuns: 0, terminalRuns: 0,
|
||||
lastStatuses: [] as Array<{ runId: string; status: string }>, complete: false, censored: false } : undefined;
|
||||
if (h5) active.terminalObservation = terminalObservation;
|
||||
try { await expect.poll(async () => {
|
||||
runs = (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId));
|
||||
if (terminalObservation) Object.assign(terminalObservation, { lastObservedAtMs: Date.now(), observedRuns: runs.length,
|
||||
terminalRuns: runs.filter(run => ["succeeded", "cancelled", "failed"].includes(run.status)).length,
|
||||
lastStatuses: runs.map(run => ({ runId: run.id, status: run.status })) });
|
||||
return runs.length === issueIds.length && runs.every(run => ["succeeded", "cancelled", "failed"].includes(run.status));
|
||||
}, { timeout: 180_000 }).toBe(true); }
|
||||
let settlement;
|
||||
try { settlement = await observeRunSettlement({ expectedRuns: issueIds.length,
|
||||
readRuns: async () => (await json(await request.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`))).filter((run: { issueId?: string; nativeIssueId?: string; contextSnapshot?: { issueId?: string; taskId?: string } }) => issueIds.includes(run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId)),
|
||||
readRun: async (runId: string) => json(await request.get(`/api/heartbeat-runs/${runId}`)),
|
||||
poll: (read: () => Promise<boolean>, milliseconds: number) => expect.poll(read, { timeout: milliseconds }).toBe(true),
|
||||
onOutcome: (value: Record<string, unknown>) => { active.terminalObservation = value; },
|
||||
}); }
|
||||
catch (error) {
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = Date.now(); terminalObservation.censored = true;
|
||||
active.terminalOriginal = originalTerminal(streamingObservation!, terminalObservation); }
|
||||
if (h5) active.terminalOriginal = originalTerminal(streamingObservation!, active.terminalObservation);
|
||||
throw error;
|
||||
}
|
||||
const settledAt = Date.now(), seenMarkers = await observed();
|
||||
if (terminalObservation) { terminalObservation.assessedAtMs = settledAt; terminalObservation.complete = true; }
|
||||
const terminalObservation = settlement.outcome;
|
||||
runs = settlement.runs;
|
||||
const settledAt = terminalObservation.assessedAtMs, seenMarkers = await observed();
|
||||
active.settledAt = settledAt; active.seenMarkers = seenMarkers;
|
||||
const details = [];
|
||||
for (const run of runs) details.push(await json(await request.get(`/api/heartbeat-runs/${run.id}`)));
|
||||
@@ -651,8 +661,9 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
...(h5 ? { h5: { streaming: streamingObservation, terminalObservation,
|
||||
terminalOriginal: originalTerminal(streamingObservation!, terminalObservation!),
|
||||
visibleCountByOriginalPaintDeadline: seenMarkers.filter(marker => marker.visibleMs <= streamingObservation!.original.deadlineMs).length,
|
||||
deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : {}),
|
||||
...(h5 ? { terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status })) } : {}),
|
||||
deadlineCountScope: "Original poll last count/time retained separately; paint-deadline count derives from unchanged2RAF timestamps" } } : { terminalObservation }),
|
||||
terminalStatuses: runs.map(run => ({ runId: run.id, runtimeMode: run.runtimeMode, status: run.status,
|
||||
nativePhase: run.nativePhase ?? null, errorCode: run.errorCode ?? null })),
|
||||
setupActivationConcurrency, setupReadinessStartedAtMs, readinessDeadlineMs, allReadyObservedAtMs, activations, activationWaves,
|
||||
nonce, markerCode, epochMs, settledAtMs: settledAt, stopRequestedAtMs: stopRequestedAt,
|
||||
stopAcknowledgedAtMs: acknowledgedAt, preludeProof, controllerAuthority, clockEvidence, interChunkGaps,
|
||||
@@ -689,7 +700,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
if (!rows.length) break;
|
||||
for (const row of rows) {
|
||||
const event = row.payload?.prpEvent;
|
||||
if (h5 && (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted")) resultAcceptedEvents++;
|
||||
if (row.eventType === "run.result.accepted" || event?.eventType === "run.result.accepted") resultAcceptedEvents++;
|
||||
if (event?.eventType !== "item.delta") continue;
|
||||
for (const match of String(event.payload?.text ?? "").matchAll(new RegExp(`\\[${code}([a-z0-9]{1,3})\\]`, "g"))) {
|
||||
const index = Number.parseInt(match[1], 36);
|
||||
@@ -712,16 +723,20 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
expect(new Set(deltas.map(item => item.identity)).size).toBe(deltas.length);
|
||||
expect(deltas.map(item => item.index)).toEqual(Array.from({ length: stop ? deltas.length : workload.events }, (_, index) => index));
|
||||
if (stop) for (const item of visible) expect(deltas.some(delta => delta.index === item.index)).toBe(true);
|
||||
if (h5) expect(resultAcceptedEvents).toBe(stop ? 0 : 1);
|
||||
expect(resultAcceptedEvents).toBe(stop ? 0 : 1);
|
||||
durable.push({ runId: run.id, token, markerCode: code, individualDeltas: deltas.length, uniqueSourceIdentities: deltas.length,
|
||||
matchedSourceClocks: deltas.length, ordered: true, ...(h5 ? { resultAcceptedEvents } : {}) });
|
||||
matchedSourceClocks: deltas.length, ordered: true, resultAcceptedEvents });
|
||||
}
|
||||
await expect.poll(() => [...state.streams.values()].every(stream => stream.ended || stream.aborted), { timeout: 30_000 }).toBe(true);
|
||||
expect(state.errors).toEqual([]);
|
||||
expect([...state.streams.values()].every(stream => !stream.error)).toBe(true);
|
||||
const recordedFailures = await runtimeAuthority({ appRoot, companyId: company.id,
|
||||
environmentId: caseEnvironment.id, issueIds, transport: caseTransport, recordedFailureRunIds: runs.map(run => run.id) });
|
||||
active.recordedFailures = recordedFailures;
|
||||
assertRecordedFailures(recordedFailures, { ...result, durable, runIds: runs.map(run => run.id) }, company.id);
|
||||
if (h5) {
|
||||
const verifiedAtMs = Date.now();
|
||||
Object.assign(result.h5!, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs,
|
||||
Object.assign((result as any).h5, { independentDrain: { verifiedAtMs, elapsedFromEpochMs: verifiedAtMs - epochMs,
|
||||
deltaCount: durable.reduce((count, row) => count + row.individualDeltas, 0),
|
||||
resultAcceptedCount: durable.reduce((count, row) => count + (row.resultAcceptedEvents ?? 0), 0),
|
||||
clockStreamsClosed: true,
|
||||
@@ -732,6 +747,7 @@ test("native ingestion reaches the shipped UI promptly and survives reload", asy
|
||||
hello: stream.hello, ended: stream.ended, aborted: stream.aborted, error: stream.error, partialBytes: stream.partialBytes,
|
||||
providerReadyAtMs: stream.providerReadyAtMs, clockHelloAtMs: stream.clockHelloAtMs, readinessObservedAtMs: stream.readinessObservedAtMs,
|
||||
emissions: [...stream.records.values()].sort((a, b) => a.index - b.index) })));
|
||||
Object.assign(result, { recordedFailures });
|
||||
(report.cases as unknown[]).push({ ...result, durable, persistedAfterReload: true });
|
||||
await info.attach(`${workload.name}${stop ? "-stop" : ""}-metrics`, { body: JSON.stringify(result, null, 2), contentType: "application/json" });
|
||||
pending.delete(key); clocks.delete(key); delete report.activeCase;
|
||||
|
||||
Reference in new issue
Block a user