diff --git a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md index b0e38ecb842..d4fe22a9293 100644 --- a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md +++ b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md @@ -8,9 +8,9 @@ scheduling, feature toggles, plugin readiness, environment cleanup, idle trackin hot-restart shutdown, and conservative sleep-safety checks. This work does not qualify actual database or host suspension, Cloud alarms, or cost savings. -The dependency base currently retains the interval while residual run recovery -is being completed. The assembled acceptance harness is implemented first so -that its observation boundary has a positive control before interval removal. +The dependency base retains the interval until final recovery integration and +review repairs converge. The assembled acceptance harness was implemented first +so that its observation boundary had a positive control before interval removal. ## Duty ownership @@ -64,9 +64,13 @@ backups, live operation keepalives or authorized external reconciliation disappe ## Clock and settlement -One positive control uses a real 31-second observation after actual startup -settlement. Longer windows use only child-local Date/setTimeout/setInterval -control. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The +The original pre-migration positive control recorded recurring maintenance SQL +during a real 31-second observation after actual startup settlement. Its baseline +receipt is retained; the obsolete recurring-SQL expectation is replaced by the +final enabled cases. The real-socket raw/prepared execution calibration remains +in the suite. The two enabled cases are prepared to observe 90 real seconds each +and two further hours using child-local Date/setTimeout/setInterval control. +They remain unqualified until run on the final cutover tree. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The clock freezes after readiness and real socket settlement, then advances at most 30 seconds per step, settling real I/O between steps. Parent deadlines, setImmediate and database sockets remain usable. Receipts distinguish real @@ -154,12 +158,12 @@ coordinator still holds unresolved transactions. PostgreSQL regressions cover committed and uncommitted changes, failed expiry, zero SQL/active work during drain, exactly-once resume and existing shutdown/restart behavior. -Remaining acceptance before a removal PR: final recovery integration, final recovery -first-producer paths through assembled wiring, enabled empty/configured/history -quiet including the real 90-second window, startup failure isolation and -remaining final-recovery lifecycle races, relevant contracts across the combined tree, current-head -Greptile 5/5 and required CI. No aggregate-removal acceptance or deployment claim -is made by the preliminary harness checkpoint. +Remaining acceptance before a removal PR: final reviewed recovery integration, +aggregate interval removal, both enabled empty/configured-history quiet cases, +relevant contracts across the combined tree, current-head Greptile 5/5 and +required CI. Startup failure isolation, producer/lifecycle cases and ordinary +restore/admission now have actual assembled proofs. No aggregate-removal +acceptance or deployment claim is made by this preparation checkpoint. The assembled routine proof also uses ordinary subtree cancel and restore APIs. It requires the same persisted routine run to return to `issue_created`, clears diff --git a/server/src/__tests__/helpers/maintenance/postgres-proxy.ts b/server/src/__tests__/helpers/maintenance/postgres-proxy.ts index 376a8767a65..5c96d0a3afc 100644 --- a/server/src/__tests__/helpers/maintenance/postgres-proxy.ts +++ b/server/src/__tests__/helpers/maintenance/postgres-proxy.ts @@ -68,7 +68,7 @@ export function createPostgresProtocolObserver(onQuery: (query: ObservedQuery) = export async function startMaintenancePostgresProxy(databaseUrl: string) { const upstream = new URL(databaseUrl), queries: ObservedQuery[] = [], errors: Error[] = []; - const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0 }; + const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0, Terminate: 0 }; const transport = { connectionsOpened: 0, connectionsClosed: 0, frontendBytes: 0, backendBytes: 0 }; const sockets = new Set(), observers = new Set>(); const server = createServer(client => { @@ -77,7 +77,7 @@ export async function startMaintenancePostgresProxy(databaseUrl: string) { const target = createConnection({ host: upstream.hostname, port: Number(upstream.port) || 5432 }); const observer = createPostgresProtocolObserver(query => queries.push(query), type => { if (type === "Q") frames.Query++; else if (type === "P") frames.Parse++; else if (type === "B") frames.Bind++; - else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++; + else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++; else if (type === "X") frames.Terminate++; }); observers.add(observer); for (const socket of [client, target]) { diff --git a/server/src/__tests__/maintenance-server-routes.test.ts b/server/src/__tests__/maintenance-server-routes.test.ts index c3d18dba861..36f5e98bda2 100644 --- a/server/src/__tests__/maintenance-server-routes.test.ts +++ b/server/src/__tests__/maintenance-server-routes.test.ts @@ -45,26 +45,55 @@ describe("assembled maintenance server production wiring", () => { if (!databaseUrl) { temporary = await startEmbeddedPostgresTestDatabase("maintenance-server-"); databaseUrl = temporary.connectionString; } }, 150_000); afterAll(async () => { await temporary?.cleanup(); }); - it("detects legacy aggregate recovery SQL after an empty real server boot settles", async () => { + it.each(["empty", "configured terminal history"] as const)("keeps enabled %s quiet for 90 real seconds and two qualified hourly windows", async state => { await clearDatabase(); - const server = await startMaintenanceServer(databaseUrl!); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + if (state !== "empty") { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + await fixture.insert(companies).values({ id: companyId, name: "Retained idle history", issuePrefix: "MIH" }); + await fixture.insert(agents).values({ id: agentId, companyId, name: "Idle retained agent", status: "idle", adapterType: "process", + adapterConfig: { command: process.execPath, args: ["-e", "process.exit(0)"] }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }); + await fixture.insert(issues).values({ id: issueId, companyId, title: "Retained terminal task", status: "done", assigneeAgentId: agentId }); + await fixture.insert(heartbeatRuns).values({ companyId, agentId, issueId, status: "succeeded", finishedAt: new Date(), contextSnapshot: { issueId } }); + } + const server = await startMaintenanceServer(databaseUrl!, { clock: true }); try { - const settled = await server.settled(); - expect(settled.startup.phase).toBe("ready"); - expect(settled.outgoingHttpRequests).toBe(0); - const baseline = server.proxy.queries.length, initialFrames = server.proxy.frames; - await delay(31_000); - await server.settled(); - const repeated = server.proxy.queries.slice(baseline); - expect(repeated.length).toBeGreaterThan(0); - expect(repeated.some(query => query.sql.includes('"heartbeat_runs"'))).toBe(true); + if (state !== "empty") await server.api("PATCH", "/instance/settings/experimental", { enableExternalObjects: true, enableStatusCards: true }); + await settleClock(server); await server.freeze(); + const before = await server.snapshot(), queryCount = server.proxy.queries.length, frames = server.proxy.frames, transport = server.proxy.transport; + expect(before.startup.phase).toBe("ready"); expect(before.idle.active).toBe(0); + await server.realtime(); + const wallStart = Date.now(); + await delay(90_000); + const wallElapsedMs = Date.now() - wallStart; + await server.freeze(); + const real = await server.settled(100); + expect(wallElapsedMs).toBeGreaterThanOrEqual(90_000); + expect(real.now - before.now).toBeGreaterThanOrEqual(89_000); + expect(real.idle.active).toBe(0); + expect(server.proxy.queries.length).toBe(queryCount); + expect(real.outgoingHttpRequests).toBe(before.outgoingHttpRequests); + expect(real.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections); + expect(real.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten); + const realFrames = server.proxy.frames, realTransport = server.proxy.transport; + for (let step = 0; step < 240; step++) { await server.advance(30_000); await server.settled(10); } + const after = await server.snapshot(); + expect(after.now - real.now).toBe(2 * 60 * 60_000); + expect(after.idle.active).toBe(0); + expect(server.proxy.queries.length).toBe(queryCount); + expect(after.outgoingHttpRequests).toBe(before.outgoingHttpRequests); + expect(after.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections); + expect(after.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten); expect(server.proxy.errors).toEqual([]); - expect((await server.snapshot()).outgoingHttpRequests).toBe(0); - await writeFile(path.join(server.directory, "measurement.json"), JSON.stringify({ phase: "legacy-positive-control", - observationMs: 31_000, startupQueries: baseline, startup: receipt(server.proxy.queries.slice(0, baseline)), - repeatedQueries: repeated.length, repeated: receipt(repeated), initialFrames, finalFrames: server.proxy.frames, outgoingHttpRequests: 0 }, null, 2)); - } finally { await server.stop(); } - }, 150_000); + await writeFile(path.join(server.directory, "enabled-quiet-measurement.json"), JSON.stringify({ phase: "enabled-quiet", state, + wallElapsedMs, realClockElapsedMs: real.now - before.now, qualifiedClockMs: after.now - real.now, stepMs: 30_000, ioSettledBetweenSteps: true, + measuredQueries: 0, startupQueries: queryCount, startup: receipt(server.proxy.queries), before, real, after, + frames: { before: frames, real: realFrames, after: server.proxy.frames }, transport: { before: transport, real: realTransport, after: server.proxy.transport }, + boundary: "actual startServer, real elapsed 90s plus qualified child clock; pool connection housekeeping reported separately; no active provider/native work, Cloud enrollment, installed plugins, telemetry exporter, announcements or database backup; no host or database suspension claim", + }, null, 2)); + } finally { await server.stop(); await fixture.$client.end(); } + }, 240_000); + it("releases an ordinary held task Retry into one real process execution", async () => { await clearDatabase(); const server = await startMaintenanceServer(databaseUrl!, { clock: true });