diff --git a/tests/restart-chaos/SCENARIOS.md b/tests/restart-chaos/SCENARIOS.md index 55d0ef0fcf7..7787077e21b 100644 --- a/tests/restart-chaos/SCENARIOS.md +++ b/tests/restart-chaos/SCENARIOS.md @@ -1686,6 +1686,8 @@ server's pid, start, exit and freezes). A violation fails the wave. `controller_lease_lost` counts only when no freeze past the lease came before it. - `MS_double_executor`: a run recorded a second local runner identity (`native.process_identity_recorded`) before the first was shown stopped (`native.local_process_stopped`). + Sandbox identities (`targetKind: "remote"`) are not graded: the server records stop evidence only for + local processes, so every in-run sandbox relaunch would look like a second executor. - Ownership failures whose owner is not a wave server (or is unknown) are listed as `unattributed`. The usual R1–R6, U1–U4 and T checks grade every task as before. In the speed grade, MS1 and MS2 take no diff --git a/tests/restart-chaos/multi-server-db.test.ts b/tests/restart-chaos/multi-server-db.test.ts index a620f392676..6efe49a2f89 100644 --- a/tests/restart-chaos/multi-server-db.test.ts +++ b/tests/restart-chaos/multi-server-db.test.ts @@ -33,6 +33,9 @@ describe.skipIf(!support.supported)("reading cross-server evidence from a wave d await db.$client.unsafe(`insert into heartbeat_runs values ('00000000-0000-4000-8000-0000000000a2', $1, null, 'succeeded', null, $2, $2, null)`, [COMPANY, iso(NOON)]); await db.$client.unsafe(`insert into heartbeat_run_events values ('00000000-0000-4000-8000-0000000000a2', 'native.process_identity_recorded', '{"processPid": 11}', $1), ('00000000-0000-4000-8000-0000000000a2', 'native.process_identity_recorded', '{"processPid": 12}', $2)`, [iso(NOON - 2_000), iso(NOON - 1_000)]); + await db.$client.unsafe(`insert into heartbeat_runs values ('00000000-0000-4000-8000-0000000000a3', $1, null, 'succeeded', null, $2, $2, null)`, [COMPANY, iso(NOON)]); + await db.$client.unsafe(`insert into heartbeat_run_events values ('00000000-0000-4000-8000-0000000000a3', 'native.process_identity_recorded', '{"processPid": 21, "targetKind": "remote"}', $1), + ('00000000-0000-4000-8000-0000000000a3', 'native.process_identity_recorded', '{"processPid": 22, "targetKind": "remote"}', $2)`, [iso(NOON - 2_000), iso(NOON - 1_000)]); }, 120_000); afterAll(async () => { @@ -45,6 +48,7 @@ describe.skipIf(!support.supported)("reading cross-server evidence from a wave d expect(evidence.runs).toEqual([expect.objectContaining({ runId: "00000000-0000-4000-8000-0000000000a1", status: "running", errorCode: "native_execution_ownership_unverified", endedAt: NOON, ownerBootId: "boot-a", ownerPid: 100 })]); expect(evidence.events.map(event => event.pid)).toEqual(expect.arrayContaining([11, 12])); + expect(evidence.events.filter(event => event.runId.endsWith("a3")).map(event => event.targetKind)).toEqual(["remote", "remote"]); const windows: ServerProcessWindow[] = [ { pid: 100, role: "primary", spawnedAt: NOON - 60_000, exitedAt: null, frozen: [{ from: NOON - 50_000, to: NOON - 5_000 }] }, { pid: 200, role: "secondary", spawnedAt: NOON - 55_000, exitedAt: null, frozen: [] }, diff --git a/tests/restart-chaos/multi-server.test.ts b/tests/restart-chaos/multi-server.test.ts index eda735f23c2..bbbd376f579 100644 --- a/tests/restart-chaos/multi-server.test.ts +++ b/tests/restart-chaos/multi-server.test.ts @@ -145,4 +145,11 @@ describe("checkCrossServer", () => { // The same runner recorded again (a re-attach) is one executor. expect(checkCrossServer({ runs, events: [recorded(1, 1), recorded(1, 5)], windows: [A] }).violations).toEqual([]); }); + + it("does not charge sandbox runner relaunches, which never record local stop evidence", () => { + const remote = (pid: number, at: number) => ({ runId: "r1", kind: "recorded" as const, pid, at, targetKind: "remote" as const }); + const runs = [run({ errorCode: null, status: "succeeded" })]; + expect(checkCrossServer({ runs, events: [remote(65122, 1), remote(57393, 5), remote(49664, 9)], windows: [A] }).violations) + .toEqual([]); + }); }); diff --git a/tests/restart-chaos/multi-server.ts b/tests/restart-chaos/multi-server.ts index 6c31c1ade74..2c71ba6b79b 100644 --- a/tests/restart-chaos/multi-server.ts +++ b/tests/restart-chaos/multi-server.ts @@ -173,7 +173,7 @@ export interface CrossServerRun { ownerPid: number | null; } -export interface RunIdentityEvent { runId: string; kind: "recorded" | "stopped"; pid: number; at: number } +export interface RunIdentityEvent { runId: string; kind: "recorded" | "stopped"; pid: number; at: number; targetKind?: "local" | "remote" } export interface CrossServerViolation { rule: "MS_foreign_failure" | "MS_double_executor"; @@ -228,6 +228,7 @@ export function checkCrossServer(input: { runs: readonly CrossServerRun[]; event } const byRun = new Map(); for (const event of input.events) { + if (event.targetKind === "remote") continue; const list = byRun.get(event.runId) ?? []; list.push(event); byRun.set(event.runId, list); @@ -268,7 +269,7 @@ export async function readCrossServerEvidence(db: Db, companyId: string): Promis [companyId]) as Row[]; const events = await client.unsafe( `select e.run_id::text as "runId", e.event_type as "eventType", e.payload->>'processPid' as pid, - floor(extract(epoch from e.created_at) * 1000) as at + e.payload->>'targetKind' as "targetKind", floor(extract(epoch from e.created_at) * 1000) as at from heartbeat_run_events e join heartbeat_runs r on r.id = e.run_id where r.company_id::text = $1 and e.event_type in ('native.process_identity_recorded', 'native.local_process_stopped')`, [companyId]) as Row[]; return { @@ -279,7 +280,8 @@ export async function readCrossServerEvidence(db: Db, companyId: string): Promis const pid = num(row.pid); const at = num(row.at); if (pid === null || at === null || !Number.isSafeInteger(pid)) return []; - return [{ runId: String(row.runId), kind: row.eventType === "native.local_process_stopped" ? "stopped" as const : "recorded" as const, pid, at }]; + return [{ runId: String(row.runId), kind: row.eventType === "native.local_process_stopped" ? "stopped" as const : "recorded" as const, pid, at, + ...(row.targetKind === "remote" ? { targetKind: "remote" as const } : {}) }]; }), }; }