test: prepare final enabled maintenance quiet windows

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 21:08:08 -05:00
1 parent 7e1c7fa1ba
commit c1bc507d27
3 files changed
+64 -31

No files matched your search

@@ -8,9 +8,9 @@ scheduling, feature toggles, plugin readiness, environment cleanup, idle trackin
hot-restart shutdown, and conservative sleep-safety checks. This work does not
qualify actual database or host suspension, Cloud alarms, or cost savings.
The dependency base currently retains the interval while residual run recovery
is being completed. The assembled acceptance harness is implemented first so
that its observation boundary has a positive control before interval removal.
The dependency base retains the interval until final recovery integration and
review repairs converge. The assembled acceptance harness was implemented first
so that its observation boundary had a positive control before interval removal.
## Duty ownership
@@ -64,9 +64,13 @@ backups, live operation keepalives or authorized external reconciliation disappe
## Clock and settlement
One positive control uses a real 31-second observation after actual startup
settlement. Longer windows use only child-local Date/setTimeout/setInterval
control. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The
The original pre-migration positive control recorded recurring maintenance SQL
during a real 31-second observation after actual startup settlement. Its baseline
receipt is retained; the obsolete recurring-SQL expectation is replaced by the
final enabled cases. The real-socket raw/prepared execution calibration remains
in the suite. The two enabled cases are prepared to observe 90 real seconds each
and two further hours using child-local Date/setTimeout/setInterval control.
They remain unqualified until run on the final cutover tree. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The
clock freezes after readiness and real socket settlement, then advances at most
30 seconds per step, settling real I/O between steps. Parent deadlines,
setImmediate and database sockets remain usable. Receipts distinguish real
@@ -154,12 +158,12 @@ coordinator still holds unresolved transactions. PostgreSQL regressions cover
committed and uncommitted changes, failed expiry, zero SQL/active work during
drain, exactly-once resume and existing shutdown/restart behavior.
Remaining acceptance before a removal PR: final recovery integration, final recovery
first-producer paths through assembled wiring, enabled empty/configured/history
quiet including the real 90-second window, startup failure isolation and
remaining final-recovery lifecycle races, relevant contracts across the combined tree, current-head
Greptile 5/5 and required CI. No aggregate-removal acceptance or deployment claim
is made by the preliminary harness checkpoint.
Remaining acceptance before a removal PR: final reviewed recovery integration,
aggregate interval removal, both enabled empty/configured-history quiet cases,
relevant contracts across the combined tree, current-head Greptile 5/5 and
required CI. Startup failure isolation, producer/lifecycle cases and ordinary
restore/admission now have actual assembled proofs. No aggregate-removal
acceptance or deployment claim is made by this preparation checkpoint.
The assembled routine proof also uses ordinary subtree cancel and restore APIs.
It requires the same persisted routine run to return to `issue_created`, clears
@@ -68,7 +68,7 @@ export function createPostgresProtocolObserver(onQuery: (query: ObservedQuery) =
export async function startMaintenancePostgresProxy(databaseUrl: string) {
const upstream = new URL(databaseUrl), queries: ObservedQuery[] = [], errors: Error[] = [];
const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0 };
const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0, Terminate: 0 };
const transport = { connectionsOpened: 0, connectionsClosed: 0, frontendBytes: 0, backendBytes: 0 };
const sockets = new Set<Socket>(), observers = new Set<ReturnType<typeof createPostgresProtocolObserver>>();
const server = createServer(client => {
@@ -77,7 +77,7 @@ export async function startMaintenancePostgresProxy(databaseUrl: string) {
const target = createConnection({ host: upstream.hostname, port: Number(upstream.port) || 5432 });
const observer = createPostgresProtocolObserver(query => queries.push(query), type => {
if (type === "Q") frames.Query++; else if (type === "P") frames.Parse++; else if (type === "B") frames.Bind++;
else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++;
else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++; else if (type === "X") frames.Terminate++;
});
observers.add(observer);
for (const socket of [client, target]) {
@@ -45,26 +45,55 @@ describe("assembled maintenance server production wiring", () => {
if (!databaseUrl) { temporary = await startEmbeddedPostgresTestDatabase("maintenance-server-"); databaseUrl = temporary.connectionString; }
}, 150_000);
afterAll(async () => { await temporary?.cleanup(); });
it("detects legacy aggregate recovery SQL after an empty real server boot settles", async () => {
it.each(["empty", "configured terminal history"] as const)("keeps enabled %s quiet for 90 real seconds and two qualified hourly windows", async state => {
await clearDatabase();
const server = await startMaintenanceServer(databaseUrl!);
const fixture = createDb(databaseUrl!, { maxConnections: 1 });
if (state !== "empty") {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
await fixture.insert(companies).values({ id: companyId, name: "Retained idle history", issuePrefix: "MIH" });
await fixture.insert(agents).values({ id: agentId, companyId, name: "Idle retained agent", status: "idle", adapterType: "process",
adapterConfig: { command: process.execPath, args: ["-e", "process.exit(0)"] }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } });
await fixture.insert(issues).values({ id: issueId, companyId, title: "Retained terminal task", status: "done", assigneeAgentId: agentId });
await fixture.insert(heartbeatRuns).values({ companyId, agentId, issueId, status: "succeeded", finishedAt: new Date(), contextSnapshot: { issueId } });
}
const server = await startMaintenanceServer(databaseUrl!, { clock: true });
try {
const settled = await server.settled();
expect(settled.startup.phase).toBe("ready");
expect(settled.outgoingHttpRequests).toBe(0);
const baseline = server.proxy.queries.length, initialFrames = server.proxy.frames;
await delay(31_000);
await server.settled();
const repeated = server.proxy.queries.slice(baseline);
expect(repeated.length).toBeGreaterThan(0);
expect(repeated.some(query => query.sql.includes('"heartbeat_runs"'))).toBe(true);
if (state !== "empty") await server.api("PATCH", "/instance/settings/experimental", { enableExternalObjects: true, enableStatusCards: true });
await settleClock(server); await server.freeze();
const before = await server.snapshot(), queryCount = server.proxy.queries.length, frames = server.proxy.frames, transport = server.proxy.transport;
expect(before.startup.phase).toBe("ready"); expect(before.idle.active).toBe(0);
await server.realtime();
const wallStart = Date.now();
await delay(90_000);
const wallElapsedMs = Date.now() - wallStart;
await server.freeze();
const real = await server.settled(100);
expect(wallElapsedMs).toBeGreaterThanOrEqual(90_000);
expect(real.now - before.now).toBeGreaterThanOrEqual(89_000);
expect(real.idle.active).toBe(0);
expect(server.proxy.queries.length).toBe(queryCount);
expect(real.outgoingHttpRequests).toBe(before.outgoingHttpRequests);
expect(real.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections);
expect(real.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten);
const realFrames = server.proxy.frames, realTransport = server.proxy.transport;
for (let step = 0; step < 240; step++) { await server.advance(30_000); await server.settled(10); }
const after = await server.snapshot();
expect(after.now - real.now).toBe(2 * 60 * 60_000);
expect(after.idle.active).toBe(0);
expect(server.proxy.queries.length).toBe(queryCount);
expect(after.outgoingHttpRequests).toBe(before.outgoingHttpRequests);
expect(after.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections);
expect(after.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten);
expect(server.proxy.errors).toEqual([]);
expect((await server.snapshot()).outgoingHttpRequests).toBe(0);
await writeFile(path.join(server.directory, "measurement.json"), JSON.stringify({ phase: "legacy-positive-control",
observationMs: 31_000, startupQueries: baseline, startup: receipt(server.proxy.queries.slice(0, baseline)),
repeatedQueries: repeated.length, repeated: receipt(repeated), initialFrames, finalFrames: server.proxy.frames, outgoingHttpRequests: 0 }, null, 2));
} finally { await server.stop(); }
}, 150_000);
await writeFile(path.join(server.directory, "enabled-quiet-measurement.json"), JSON.stringify({ phase: "enabled-quiet", state,
wallElapsedMs, realClockElapsedMs: real.now - before.now, qualifiedClockMs: after.now - real.now, stepMs: 30_000, ioSettledBetweenSteps: true,
measuredQueries: 0, startupQueries: queryCount, startup: receipt(server.proxy.queries), before, real, after,
frames: { before: frames, real: realFrames, after: server.proxy.frames }, transport: { before: transport, real: realTransport, after: server.proxy.transport },
boundary: "actual startServer, real elapsed 90s plus qualified child clock; pool connection housekeeping reported separately; no active provider/native work, Cloud enrollment, installed plugins, telemetry exporter, announcements or database backup; no host or database suspension claim",
}, null, 2));
} finally { await server.stop(); await fixture.$client.end(); }
}, 240_000);
it("releases an ordinary held task Retry into one real process execution", async () => {
await clearDatabase();
const server = await startMaintenanceServer(databaseUrl!, { clock: true });