mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
test: correct blocked Stop provider groups and cleanup order
Record the provider process group independently and prove both owned groups alive through the hold and absent after Stop. Request supported public cancellation before closing the fixture clock and retain cleanup outcomes. Keep the workload, precision budget and original deadlines unchanged.
This commit is contained in:
1 parent
200396d5b7
commit
6e5349cdbd
4 files changed
+167
-33
No files matched your search
@@ -12,14 +12,51 @@ export function isAppendLockQuery(query) {
|
||||
return /^select "company_id", "agent_id"(?:,| from)/i.test(sql)
|
||||
&& / from "heartbeat_runs" where "heartbeat_runs"\."id" = \$\d+.* for update/i.test(sql);
|
||||
}
|
||||
export function assertHeldSample(sample, holderPid) {
|
||||
export function assertHeldSample(sample, holderPid, expectedProcesses) {
|
||||
assert.equal(sample.httpPending, true, 'Stop HTTP must remain pending while its durable barrier is locked');
|
||||
assert.equal(sample.intent, null, 'No committed intent before releasing the run lock');
|
||||
assert.deepEqual(sample.auditIds, [], 'No committed cancellation audit before releasing the run lock');
|
||||
assert.equal(sample.processesAlive, true, 'No physical provider Stop before the durable intent barrier');
|
||||
assert(expectedProcesses?.runner && expectedProcesses.provider, 'Missing original owned process births');
|
||||
assert.deepEqual(sample.processes?.runner, expectedProcesses.runner);
|
||||
assert.deepEqual(sample.processes?.provider, expectedProcesses.provider);
|
||||
assert.equal(sample.processes?.group, expectedProcesses.group);
|
||||
assert.equal(sample.processes?.providerGroup, expectedProcesses.providerGroup);
|
||||
assert.equal(sample.processes?.runnerGroupAlive, true);
|
||||
assert.equal(sample.processes?.providerGroupAlive, true);
|
||||
assert.equal(sample.clockOpen, true, 'The unchanged provider fixture closes its clock on turn/interrupt');
|
||||
assert(sample.blockers.some(row => row.blocking_pids.includes(holderPid)), 'Missing independent PostgreSQL blocker');
|
||||
}
|
||||
// Public cancellation supports active/scheduled runs and native retryable failures.
|
||||
// A receipt is not physical Stop proof; the independent process/lease oracle remains required.
|
||||
export function cleanupCancellationEligible(run) {
|
||||
return ['queued', 'running', 'scheduled_retry'].includes(run.status)
|
||||
|| (run.status === 'failed' && run.runtimeMode === 'native' && run.nativePhase === 'retryable_failure');
|
||||
}
|
||||
export async function cleanupOwnedRunBeforeClockClose({ companyId, runId, readRun, cancelRun, closeClock, record }) {
|
||||
const snapshot = run => {
|
||||
assert.equal(run.id, runId); assert.equal(run.companyId, companyId);
|
||||
return { id: run.id, companyId: run.companyId, status: run.status, runtimeMode: run.runtimeMode,
|
||||
nativePhase: run.nativePhase ?? null, errorCode: run.errorCode ?? null,
|
||||
nativeCancellation: run.resultJson?.nativeCancellation ?? null };
|
||||
};
|
||||
try {
|
||||
if (readRun) {
|
||||
const before = await readRun(); record.before = snapshot(before);
|
||||
record.cancel = { attempted: cleanupCancellationEligible(before) };
|
||||
if (record.cancel.attempted) {
|
||||
const response = await cancelRun();
|
||||
record.cancel.httpStatus = response.status; record.cancel.response = snapshot(response.run);
|
||||
}
|
||||
record.after = snapshot(await readRun());
|
||||
}
|
||||
} catch (error) { record.error = String(error); throw error; }
|
||||
finally {
|
||||
record.clockCloseStartedAtMs = Date.now();
|
||||
try { await closeClock(); record.clockClosedAtMs = Date.now(); }
|
||||
catch (error) { record.clockCloseError = String(error); throw error; }
|
||||
}
|
||||
}
|
||||
export function assertIntentAudit(row) {
|
||||
const intent = row.intent;
|
||||
assert.equal(intent?.schema, 'paperclip.native-cancellation.v1');
|
||||
@@ -250,7 +287,7 @@ export function assertFunctional(e) {
|
||||
assert(e.appendBlockedBeforeStop.rows.some(row => isAppendLockQuery(row.query) && row.blocking_pids.includes(e.lock.holder_pid)));
|
||||
assert(e.heldSamples?.length >= 2);
|
||||
for (const sample of e.heldSamples) {
|
||||
assertHeldSample(sample, e.lock.holder_pid); assertOptionalShared(sample, e.companyId, e.runId);
|
||||
assertHeldSample(sample, e.lock.holder_pid, e.processesBefore); assertOptionalShared(sample, e.companyId, e.runId);
|
||||
assert(sample.queryStartedAtMs >= e.timing.requestAtMs && sample.observedAtMs <= e.timing.releaseRequestedAtMs);
|
||||
}
|
||||
assert(e.heldSamples.at(-1).emittedCount > e.heldSamples[0].emittedCount, 'Provider must still emit through the stall');
|
||||
@@ -265,7 +302,9 @@ export function assertFunctional(e) {
|
||||
}
|
||||
assert.equal(e.ack.company_id, e.companyId); assert.equal(e.ack.run_id, e.runId); assert.equal(e.ack.issue_id, e.issueId);
|
||||
assert.equal(e.stopHttpStatus, 200);
|
||||
assert.deepEqual([e.processesAfter?.runnerAbsent, e.processesAfter?.groupAbsent, e.processesAfter?.providerAbsent], [true,true,true]);
|
||||
assert.deepEqual([e.processesAfter?.runnerAbsent, e.processesAfter?.groupAbsent, e.processesAfter?.providerAbsent, e.processesAfter?.providerGroupAbsent], [true,true,true,true]);
|
||||
assert.equal(e.processesAfter.group, e.processesBefore.group);
|
||||
assert.equal(e.processesAfter.providerGroup, e.processesBefore.providerGroup);
|
||||
assert(e.timing.httpSettledAtMs >= e.timing.releaseRequestedAtMs);
|
||||
const before = e.authorityBefore?.rows?.[0], after = e.authorityAfter?.rows?.[0];
|
||||
for (const row of [before, after]) { assert.equal(row?.company_id, e.companyId); assert.equal(row?.run_id, e.runId); assert.equal(row.runtime_mode, 'native'); }
|
||||
@@ -276,8 +315,12 @@ export function assertFunctional(e) {
|
||||
assert.equal(after.process_stopped.processGroupId, e.processesBefore.group);
|
||||
for (const process of [e.processesBefore.runner, e.processesBefore.provider]) {
|
||||
assert(Number.isSafeInteger(process.pid) && process.pid > 1 && process.birth);
|
||||
assert.equal(process.group, e.processesBefore.group);
|
||||
assert(Number.isSafeInteger(process.group) && process.group > 1);
|
||||
}
|
||||
assert.equal(e.processesBefore.runner.group, e.processesBefore.group);
|
||||
assert.equal(e.processesBefore.group, before.process_group_id);
|
||||
assert.equal(e.processesBefore.runner.pid, before.process_pid);
|
||||
assert.equal(e.processesBefore.provider.group, e.processesBefore.providerGroup);
|
||||
assert.equal(e.lease?.heartbeatRunId, e.runId); assert.equal(e.lease.companyId, e.companyId);
|
||||
assert.equal(e.lease.provider, 'local'); assert.equal(e.lease.providerLeaseId, null);
|
||||
assert(e.lease.releasedAt && ['released','expired','failed'].includes(e.lease.status)); assert.notEqual(e.lease.cleanupStatus, 'failed');
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { PROTOCOL, WORKLOAD as W, WORKLOAD_SHA256, METRICS, isAppendLockQuery, assertHeldSample,
|
||||
assertIntentAudit, metrics, registration, noiseEnvelope, comparePairs, assertFunctional, observerBrackets, assertNoiseEnvelope, PRECISION_RULE } from './native-ingestion-blocked-stop-protocol.mjs';
|
||||
assertIntentAudit, metrics, registration, noiseEnvelope, comparePairs, assertFunctional, observerBrackets, assertNoiseEnvelope, PRECISION_RULE, cleanupCancellationEligible, cleanupOwnedRunBeforeClockClose } from './native-ingestion-blocked-stop-protocol.mjs';
|
||||
const evidence = (shift = 0) => ({ protocol: PROTOCOL, workloadSha256: WORKLOAD_SHA256, complete: true, cleanup: { complete: true },
|
||||
timing: { actionAtMs: 1000, requestAtMs: 1005, releaseRequestedAtMs: 2005, releaseCommitAckAtMs: 2007,
|
||||
absenceObservedAtMs: 2030 + shift, ackObservedAtMs: 2040 + shift } });
|
||||
const sample = () => ({ httpPending: true, intent: null, auditIds: [], processesAlive: true, clockOpen: true, blockers: [{ blocking_pids: [88] }] });
|
||||
const processes = () => ({ runner: { pid: 10, group: 10, birth: 'runner-birth' }, provider: { pid: 11, group: 11, birth: 'provider-birth' }, group: 10, providerGroup: 11 });
|
||||
const sample = () => ({ processes: { ...processes(), runnerGroupAlive: true, providerGroupAlive: true }, httpPending: true, intent: null, auditIds: [], processesAlive: true, clockOpen: true, blockers: [{ blocking_pids: [88] }] });
|
||||
const intentRow = () => ({ company_id: 'c', run_id: 'r', issue_id: 'i', intent: {
|
||||
schema: 'paperclip.native-cancellation.v1', companyId: 'c', runId: 'r', issueId: 'i',
|
||||
intentId: 'own', intentAuditId: 'a', dispatchState: 'acknowledged', acknowledgementAuditId: 'b',
|
||||
@@ -25,10 +26,10 @@ test('B0 and candidate append SQL match while cancellation full-row and unrelate
|
||||
assert(!isAppendLockQuery('select "company_id", "agent_id" from "heartbeat_runs" where "heartbeat_runs"."id" = $1'));
|
||||
});
|
||||
test('held sample requires every barrier and physical liveness proof', () => {
|
||||
assertHeldSample(sample(), 88);
|
||||
assertHeldSample(sample(), 88, processes());
|
||||
for (const patch of [{ httpPending: false }, { intent: {} }, { auditIds: ['a'] }, { processesAlive: false }, { clockOpen: false }, { blockers: [] }])
|
||||
assert.throws(() => assertHeldSample({ ...sample(), ...patch }, 88));
|
||||
assert.throws(() => assertHeldSample(sample(), 89));
|
||||
assert.throws(() => assertHeldSample({ ...sample(), ...patch }, 88, processes()));
|
||||
assert.throws(() => assertHeldSample(sample(), 89, processes()));
|
||||
});
|
||||
test('ACK correlates both committed audits to the same owned intent', () => {
|
||||
assertIntentAudit(intentRow());
|
||||
@@ -56,7 +57,7 @@ test('registered companion is 50 separate alternating arms with fresh B0 and B1
|
||||
});
|
||||
|
||||
const functional = () => {
|
||||
const e = evidence(), identity = { company_id: 'c', run_id: 'r', runtime_mode: 'native' };
|
||||
const e = evidence(), identity = { company_id: 'c', run_id: 'r', runtime_mode: 'native', process_pid: 10, process_group_id: 10 };
|
||||
return { ...e, variant: 'b0', startedAtMs: 100, finishedAtMs: 3000, companyId: 'c', runId: 'r', issueId: 'i', epochMs: 900,
|
||||
prelude: { individualDeltas: 1000, uniqueIdentities: 1000, verifiedBeforeEpochAtMs: 850 },
|
||||
visibleBeforeLock: [0,1,2], lock: { id: 'r', company_id: 'c', holder_pid: 88 },
|
||||
@@ -67,8 +68,7 @@ const functional = () => {
|
||||
observerSamples: [{ kind: 'ack-query', startedAtMs: 2035, completedAtMs: 2036, durationMs: 1, value: false },
|
||||
{ kind: 'ack-query', startedAtMs: 2039, completedAtMs: 2040, durationMs: 1, value: true },
|
||||
{ kind: 'process-check', startedAtMs: 2025, completedAtMs: 2026, durationMs: 1, value: false },
|
||||
{ kind: 'process-check', startedAtMs: 2029, completedAtMs: 2030, durationMs: 1, value: true }], processesBefore: { runner: { pid: 10, group: 10, birth: 'birth' },
|
||||
provider: { pid: 11, group: 10, birth: 'birth' }, group: 10 }, processesAfter: { runnerAbsent: true, groupAbsent: true, providerAbsent: true },
|
||||
{ kind: 'process-check', startedAtMs: 2029, completedAtMs: 2030, durationMs: 1, value: true }], processesBefore: processes(), processesAfter: { runnerAbsent: true, groupAbsent: true, providerAbsent: true, providerGroupAbsent: true, group: 10, providerGroup: 11 },
|
||||
authorityBefore: { rows: [{ ...identity, process_identity_seq: '1' }] },
|
||||
authorityAfter: { rows: [{ ...identity, status: 'cancelled', cancellation_dispatch_state: 'acknowledged', process_stopped_seq: '20',
|
||||
process_stopped_source_event_id: null, process_stopped: { processPid: 10, processGroupId: 10 } }] },
|
||||
@@ -204,3 +204,51 @@ test('calibration and comparison reject reused runs, overlapping arms and nonalt
|
||||
assert.throws(() => noiseEnvelope(invalid)); assert.throws(() => comparePairs(invalid, noise));
|
||||
}
|
||||
});
|
||||
|
||||
test('distinct provider group is valid, same-group topology is also valid, and both exact owned groups must be proved', () => {
|
||||
const distinct = functional(); assert.notEqual(distinct.processesBefore.group, distinct.processesBefore.providerGroup); assertFunctional(distinct);
|
||||
const shared = functional(); shared.processesBefore.provider.group = 10; shared.processesBefore.providerGroup = 10;
|
||||
shared.processesAfter.providerGroup = 10;
|
||||
for (const held of shared.heldSamples) { held.processes.provider.group = 10; held.processes.providerGroup = 10; }
|
||||
assertFunctional(shared);
|
||||
for (const mutate of [e => { delete e.processesBefore.providerGroup; }, e => { e.processesBefore.providerGroup = 99; },
|
||||
e => { e.processesBefore.provider.group = 0; }, e => { delete e.processesAfter.providerGroupAbsent; },
|
||||
e => { e.processesAfter.providerGroupAbsent = false; }, e => { e.processesAfter.providerGroup = 99; },
|
||||
e => { e.authorityBefore.rows[0].process_group_id = 99; }, e => { e.processesBefore.runner.group = 99; },
|
||||
e => { e.heldSamples[1].processes.provider.birth = 'different-birth'; }, e => { e.heldSamples[1].processes.runner.pid = 99; },
|
||||
e => { e.heldSamples[1].processes.providerGroupAlive = false; }, e => { e.heldSamples[1].processes.runnerGroupAlive = false; },
|
||||
e => { delete e.heldSamples[1].processes; }]) {
|
||||
const e = functional(); mutate(e); assert.throws(() => assertFunctional(e));
|
||||
}
|
||||
});
|
||||
test('cleanup eligibility follows supported active/scheduled and native retryable phases only', () => {
|
||||
for (const status of ['queued','running','scheduled_retry']) assert(cleanupCancellationEligible({ status }));
|
||||
assert(cleanupCancellationEligible({ status:'failed', runtimeMode:'native', nativePhase:'retryable_failure' }));
|
||||
for (const run of [{ status:'failed', runtimeMode:'legacy', nativePhase:'retryable_failure' },
|
||||
{ status:'failed', runtimeMode:'native', nativePhase:'terminal_failure' }, { status:'failed', runtimeMode:'native' },
|
||||
{ status:'succeeded' }, { status:'cancelled' }, { status:'timed_out' }]) assert.equal(cleanupCancellationEligible(run), false);
|
||||
});
|
||||
test('owned cleanup records public cancellation before closing its clock, including retryable failure', async () => {
|
||||
for (const before of [{ status:'running', runtimeMode:'native' }, { status:'failed', runtimeMode:'native', nativePhase:'retryable_failure' }]) {
|
||||
const order=[],record={}; let open=true,run={ id:'r',companyId:'c',...before };
|
||||
await cleanupOwnedRunBeforeClockClose({ companyId:'c',runId:'r',record,
|
||||
readRun:async()=>{ assert(open);order.push('read');return run; },
|
||||
cancelRun:async()=>{ assert(open);order.push('cancel');run={...run,status:'cancelled',resultJson:{nativeCancellation:{dispatchState:'acknowledged'}}};return {status:200,run}; },
|
||||
closeClock:async()=>{ order.push('close');open=false; } });
|
||||
assert.deepEqual(order,['read','cancel','read','close']); assert.equal(record.before.status,before.status);
|
||||
assert.equal(record.cancel.httpStatus,200); assert.equal(record.after.status,'cancelled'); assert.equal(record.after.nativeCancellation.dispatchState,'acknowledged');
|
||||
assert.equal('providerAbsent' in record,false,'Cancellation response is not physical Stop evidence');
|
||||
}
|
||||
});
|
||||
test('cleanup does not cancel terminal/foreign runs and closes clock after a failed cancellation attempt', async () => {
|
||||
for (const mode of ['terminal','foreign','cancel-error']) {
|
||||
const order=[],record={};let cancelled=false;
|
||||
const action=()=>cleanupOwnedRunBeforeClockClose({companyId:'c',runId:'r',record,
|
||||
readRun:async()=>{order.push('read');return {id:'r',companyId:mode==='foreign'?'other':'c',status:mode==='terminal'?'succeeded':'running'};},
|
||||
cancelRun:async()=>{cancelled=true;order.push('cancel');throw Error('controlled cancellation error');},
|
||||
closeClock:async()=>{order.push('close');} });
|
||||
if(mode==='terminal'){await action();assert.equal(record.cancel.attempted,false);assert.equal(cancelled,false);}
|
||||
else {await assert.rejects(action());assert(record.error);assert.equal(cancelled,mode==='cancel-error');}
|
||||
assert.equal(order.at(-1),'close');assert(record.clockClosedAtMs);
|
||||
}
|
||||
});
|
||||
@@ -1,10 +1,35 @@
|
||||
# Existing-history blocked Stop companion (prospective v1)
|
||||
# Existing-history blocked Stop companion (prospective instrumentation v2)
|
||||
|
||||
Status: local source and pure oracle calibration only. No application, browser,
|
||||
Status: v2 local source and pure oracle calibration only. No v2 application, browser,
|
||||
PostgreSQL, VM or quantitative arm has run. No runtime qualification or performance
|
||||
claim follows from the pure tests. Execution needs lead release after independent
|
||||
source review. This is separate from the immutable H5c 50-arm campaign.
|
||||
|
||||
## Preserved v1 qualification failure and v2 correction
|
||||
|
||||
The one v1 qualification at `200396d5` failed before the emission epoch because
|
||||
the probe assumed the provider shared the Runner process group. Actual Runner
|
||||
PID/group was 2284214; provider PID/group was 2284218. Separate groups are allowed.
|
||||
No held-append, Stop timing or calibration data was collected. The failed raw
|
||||
qualification remains retained and cannot supply a v2 timing arm or noise sample.
|
||||
|
||||
Normal API cleanup returned 200 and the port was released, but strict physical
|
||||
quiet failed: both exact owned processes remained sleeping after the app closed.
|
||||
The old finally block closed the clock before cancelling the run and did not
|
||||
attempt cancellation for failed native retryable phases. Those observations do not establish a product
|
||||
cause or successful physical termination from the API response.
|
||||
|
||||
This v2 source records independent provider PID birth and process group, proves
|
||||
both groups alive during the hold and absent after Stop, and keeps the server
|
||||
termination receipt correlated to the Runner group. Every held sample retains
|
||||
both original process births. Shared-group topology remains valid when observed.
|
||||
Cleanup releases its owned lock, attempts supported public cancellation while the
|
||||
clock is still available, records before/HTTP response/after status and native
|
||||
cancellation metadata, and then closes the clock. No manual process kill belongs
|
||||
to a passed functional oracle; exact owned cleanup after a failed qualification
|
||||
is separate negative evidence. New source review and runtime qualification are
|
||||
required. Protocol name, workload, precision rule and all deadlines are unchanged.
|
||||
|
||||
## Registration and ownership
|
||||
|
||||
Explicit config: `tests/e2e/playwright-native-ingestion-blocked-stop.config.ts`.
|
||||
@@ -54,7 +79,7 @@ then require the cancellation intent and its own matching committed audit,
|
||||
native acknowledged dispatch and its own matching acknowledgement audit,
|
||||
normal cancelled settlement, successful Stop HTTP, server-authored
|
||||
`native.local_process_stopped` receipt for the exact prior PID/group, independently
|
||||
absent runner/group/provider, and the exact owned local lease released without
|
||||
absent runner/group/provider/provider-group, and the exact owned local lease released without
|
||||
failed cleanup. Transcript checks require complete compact payloads, unique source
|
||||
identities, ordered canonical prefix, original emission clocks, zero accepted
|
||||
results, and persistence after reload. The raw transport recovery policy stays
|
||||
@@ -86,8 +111,15 @@ uncertainty. Screenshots and reload happen after those timing observations.
|
||||
Any failure, timeout, censored observation, query/clock fault or cleanup error
|
||||
makes the arm ineligible. Partial raw Playwright results and the final attachment
|
||||
remain even after failure. Cleanup first releases/rolls back only the owned lock,
|
||||
closes both connections, retires its clock, settles only its owned run and restores
|
||||
owned experimental settings. The external operator must independently prove the
|
||||
closes both connections, settles only its owned run while its clock is available,
|
||||
then retires the clock and restores owned experimental settings. Supported public
|
||||
phases are queued/running/scheduled_retry and failed native retryable_failure;
|
||||
other terminal phases are not presumed cancelable or physically stopped. This
|
||||
cleanup subset follows `server/src/services/heartbeat/recovery.ts` active statuses
|
||||
and `native-cancellation-request.ts` retryable eligibility, reached through the
|
||||
normal board POST in `routes/agents.ts`. The server rechecks durable coordinator
|
||||
eligibility and owns cancellation barriers; the probe does not grant authority.
|
||||
The external operator must independently prove the
|
||||
server, database, runner, provider and browser are gone before the next arm.
|
||||
|
||||
## Separate quantitative registration (50 short arms)
|
||||
|
||||
@@ -7,7 +7,7 @@ import { json } from "./agent-chat.shared";
|
||||
import { runtimeAuthority, processAbsent } from "./fixtures/native-ingestion-blocked-stop-authority";
|
||||
import { openClock } from "./fixtures/native-ingestion-blocked-stop-clock";
|
||||
import { ownedDatabase, cancellationSnapshot, blockers } from "./fixtures/native-ingestion-blocked-stop-db";
|
||||
import { PROTOCOL, WORKLOAD as W, WORKLOAD_SHA256, assertHeldSample, assertIntentAudit, isAppendLockQuery, metrics, assertFunctional }
|
||||
import { PROTOCOL, WORKLOAD as W, WORKLOAD_SHA256, assertHeldSample, assertIntentAudit, isAppendLockQuery, metrics, assertFunctional, cleanupOwnedRunBeforeClockClose }
|
||||
from "./fixtures/native-ingestion-blocked-stop-protocol.mjs";
|
||||
import { observeRunSettlement } from "./fixtures/native-ingestion-h5.mjs";
|
||||
|
||||
@@ -143,9 +143,12 @@ test("existing-history Stop preserves its durable barrier behind a real blocked
|
||||
expect(row.run_id).toBe(run.id);
|
||||
expect(row.process_identity.processPid).toBe(row.process_pid);
|
||||
expect(row.process_identity.processGroupId).toBe(row.process_group_id);
|
||||
const processes = { runner: birth(row.process_pid), provider: birth(stream.providerPid), group: row.process_group_id };
|
||||
expect(processes.runner.group).toBe(processes.group); expect(processes.provider.group).toBe(processes.group);
|
||||
const alive = () => !processAbsent(processes.runner.pid) && !processAbsent(-processes.group) && !processAbsent(processes.provider.pid);
|
||||
const provider = birth(stream.providerPid);
|
||||
const processes = { runner: birth(row.process_pid), provider, group: row.process_group_id, providerGroup: provider.group };
|
||||
expect(processes.runner.group).toBe(processes.group);
|
||||
expect(Number.isSafeInteger(processes.providerGroup) && processes.providerGroup > 1).toBe(true);
|
||||
const alive = () => !processAbsent(processes.runner.pid) && !processAbsent(-processes.group)
|
||||
&& !processAbsent(processes.provider.pid) && !processAbsent(-processes.providerGroup);
|
||||
expect(alive()).toBe(true); report.authorityBefore = authority; report.processesBefore = processes;
|
||||
state.epochMs = Date.now() + 1000; report.epochMs = state.epochMs;
|
||||
clock.pending.get(key)!.get(nonce)!.writeHead(200, { "Content-Type": "application/json" }).end(JSON.stringify({ epochMs: state.epochMs }));
|
||||
@@ -177,11 +180,12 @@ test("existing-history Stop preserves its durable barrier behind a real blocked
|
||||
do {
|
||||
const snapshot = await cancellationSnapshot(observer, company.id, run.id);
|
||||
const sample = { ...snapshot, httpPending: !stopSettled, auditIds: snapshot.audits.map((audit: any) => audit.id),
|
||||
processesAlive: alive(), clockOpen: !stream.ended && !stream.aborted,
|
||||
processesAlive: alive(), processes: { runner: birth(processes.runner.pid), provider: birth(processes.provider.pid),
|
||||
group: processes.group, providerGroup: processes.providerGroup,
|
||||
runnerGroupAlive: !processAbsent(-processes.group), providerGroupAlive: !processAbsent(-processes.providerGroup) },
|
||||
clockOpen: !stream.ended && !stream.aborted,
|
||||
emittedCount: stream.records.size, blockers: await blockers(observer, holderPid) };
|
||||
expect(birth(processes.runner.pid)).toEqual(processes.runner);
|
||||
expect(birth(processes.provider.pid)).toEqual(processes.provider);
|
||||
report.heldSamples.push(sample); assertHeldSample(sample, holderPid);
|
||||
report.heldSamples.push(sample); assertHeldSample(sample, holderPid, processes);
|
||||
if (Date.now() >= report.timing.requestAtMs + W.heldAfterActualRequestMs) break;
|
||||
await delay(Math.min(W.pollIntervalMs, report.timing.requestAtMs + W.heldAfterActualRequestMs - Date.now()));
|
||||
} while (true);
|
||||
@@ -204,14 +208,16 @@ test("existing-history Stop preserves its durable barrier behind a real blocked
|
||||
}
|
||||
if (!absent) {
|
||||
const checkStartedAtMs = Date.now();
|
||||
const value = processAbsent(processes.runner.pid) && processAbsent(-processes.group) && processAbsent(processes.provider.pid);
|
||||
const value = processAbsent(processes.runner.pid) && processAbsent(-processes.group)
|
||||
&& processAbsent(processes.provider.pid) && processAbsent(-processes.providerGroup);
|
||||
const completedAtMs = Date.now();
|
||||
report.observerSamples.push({ kind: "process-check", startedAtMs: checkStartedAtMs,
|
||||
completedAtMs, durationMs: completedAtMs - checkStartedAtMs, value });
|
||||
if (value) {
|
||||
absent = true; report.timing.absenceObservedAtMs = completedAtMs;
|
||||
report.timing.absenceSuccessfulCheckStartedAtMs = checkStartedAtMs;
|
||||
report.processesAfter = { runnerAbsent: true, groupAbsent: true, providerAbsent: true, checkedAtMs: completedAtMs };
|
||||
report.processesAfter = { runnerAbsent: true, groupAbsent: true, providerAbsent: true, providerGroupAbsent: true,
|
||||
group: processes.group, providerGroup: processes.providerGroup, checkedAtMs: completedAtMs };
|
||||
} else report.timing.lastProcessesPresentAtMs = completedAtMs;
|
||||
}
|
||||
};
|
||||
@@ -289,12 +295,17 @@ test("existing-history Stop preserves its durable barrier behind a real blocked
|
||||
if (held) await cleanup("rollback-owned-lock", async () => { await holder.unsafe("ROLLBACK"); held = false; });
|
||||
if (holder) await cleanup("close-holder", () => holder.end({ timeout: 5 }));
|
||||
if (observer) await cleanup("close-observer", () => observer.end({ timeout: 5 }));
|
||||
for (const pending of clock.pending.values()) for (const response of pending.values()) if (!response.writableEnded) response.writeHead(503).end();
|
||||
clock.gate.closeAllConnections(); await new Promise<void>(resolve => clock.gate.close(() => resolve()));
|
||||
if (companyRunId) await cleanup("settle-owned-run", async () => {
|
||||
const run = await json(await request.get(`/api/heartbeat-runs/${companyRunId}`));
|
||||
if (["running", "queued"].includes(run.status)) await json(await request.post(`/api/heartbeat-runs/${companyRunId}/cancel`));
|
||||
});
|
||||
report.cleanupSettlement = {};
|
||||
await cleanup("settle-owned-run-before-clock-close", () => cleanupOwnedRunBeforeClockClose({
|
||||
companyId: company.id, runId: companyRunId,
|
||||
readRun: companyRunId ? async () => json(await request.get(`/api/heartbeat-runs/${companyRunId}`)) : undefined,
|
||||
cancelRun: async () => { const response = await request.post(`/api/heartbeat-runs/${companyRunId}/cancel`);
|
||||
return { status: response.status(), run: await json(response) }; },
|
||||
closeClock: async () => {
|
||||
for (const pending of clock.pending.values()) for (const response of pending.values()) if (!response.writableEnded) response.writeHead(503).end();
|
||||
clock.gate.closeAllConnections(); await new Promise<void>(resolve => clock.gate.close(() => resolve()));
|
||||
}, record: report.cleanupSettlement,
|
||||
}));
|
||||
await cleanup("restore-experimental-settings", async () => json(await request.patch("/api/instance/settings/experimental", {
|
||||
data: { enableNativeRunner: original.enableNativeRunner, enableWorktreeRunExecution: original.enableWorktreeRunExecution } })));
|
||||
report.cleanup = { complete: cleanupErrors.length === 0, errors: cleanupErrors };
|
||||
|
||||
Reference in new issue
Block a user