mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 01:27:09 +01:00
test: prepare final enabled maintenance quiet windows
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
7e1c7fa1ba
commit
c1bc507d27
3 files changed
+64
-31
No files matched your search
@@ -8,9 +8,9 @@ scheduling, feature toggles, plugin readiness, environment cleanup, idle trackin
|
||||
hot-restart shutdown, and conservative sleep-safety checks. This work does not
|
||||
qualify actual database or host suspension, Cloud alarms, or cost savings.
|
||||
|
||||
The dependency base currently retains the interval while residual run recovery
|
||||
is being completed. The assembled acceptance harness is implemented first so
|
||||
that its observation boundary has a positive control before interval removal.
|
||||
The dependency base retains the interval until final recovery integration and
|
||||
review repairs converge. The assembled acceptance harness was implemented first
|
||||
so that its observation boundary had a positive control before interval removal.
|
||||
|
||||
## Duty ownership
|
||||
|
||||
@@ -64,9 +64,13 @@ backups, live operation keepalives or authorized external reconciliation disappe
|
||||
|
||||
## Clock and settlement
|
||||
|
||||
One positive control uses a real 31-second observation after actual startup
|
||||
settlement. Longer windows use only child-local Date/setTimeout/setInterval
|
||||
control. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The
|
||||
The original pre-migration positive control recorded recurring maintenance SQL
|
||||
during a real 31-second observation after actual startup settlement. Its baseline
|
||||
receipt is retained; the obsolete recurring-SQL expectation is replaced by the
|
||||
final enabled cases. The real-socket raw/prepared execution calibration remains
|
||||
in the suite. The two enabled cases are prepared to observe 90 real seconds each
|
||||
and two further hours using child-local Date/setTimeout/setInterval control.
|
||||
They remain unqualified until run on the final cutover tree. Boot, ordinary HTTP/service producer phases and graceful shutdown advance at real speed. The
|
||||
clock freezes after readiness and real socket settlement, then advances at most
|
||||
30 seconds per step, settling real I/O between steps. Parent deadlines,
|
||||
setImmediate and database sockets remain usable. Receipts distinguish real
|
||||
@@ -154,12 +158,12 @@ coordinator still holds unresolved transactions. PostgreSQL regressions cover
|
||||
committed and uncommitted changes, failed expiry, zero SQL/active work during
|
||||
drain, exactly-once resume and existing shutdown/restart behavior.
|
||||
|
||||
Remaining acceptance before a removal PR: final recovery integration, final recovery
|
||||
first-producer paths through assembled wiring, enabled empty/configured/history
|
||||
quiet including the real 90-second window, startup failure isolation and
|
||||
remaining final-recovery lifecycle races, relevant contracts across the combined tree, current-head
|
||||
Greptile 5/5 and required CI. No aggregate-removal acceptance or deployment claim
|
||||
is made by the preliminary harness checkpoint.
|
||||
Remaining acceptance before a removal PR: final reviewed recovery integration,
|
||||
aggregate interval removal, both enabled empty/configured-history quiet cases,
|
||||
relevant contracts across the combined tree, current-head Greptile 5/5 and
|
||||
required CI. Startup failure isolation, producer/lifecycle cases and ordinary
|
||||
restore/admission now have actual assembled proofs. No aggregate-removal
|
||||
acceptance or deployment claim is made by this preparation checkpoint.
|
||||
|
||||
The assembled routine proof also uses ordinary subtree cancel and restore APIs.
|
||||
It requires the same persisted routine run to return to `issue_created`, clears
|
||||
|
||||
@@ -68,7 +68,7 @@ export function createPostgresProtocolObserver(onQuery: (query: ObservedQuery) =
|
||||
|
||||
export async function startMaintenancePostgresProxy(databaseUrl: string) {
|
||||
const upstream = new URL(databaseUrl), queries: ObservedQuery[] = [], errors: Error[] = [];
|
||||
const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0 };
|
||||
const frames = { Query: 0, Parse: 0, Bind: 0, Execute: 0, Sync: 0, Terminate: 0 };
|
||||
const transport = { connectionsOpened: 0, connectionsClosed: 0, frontendBytes: 0, backendBytes: 0 };
|
||||
const sockets = new Set<Socket>(), observers = new Set<ReturnType<typeof createPostgresProtocolObserver>>();
|
||||
const server = createServer(client => {
|
||||
@@ -77,7 +77,7 @@ export async function startMaintenancePostgresProxy(databaseUrl: string) {
|
||||
const target = createConnection({ host: upstream.hostname, port: Number(upstream.port) || 5432 });
|
||||
const observer = createPostgresProtocolObserver(query => queries.push(query), type => {
|
||||
if (type === "Q") frames.Query++; else if (type === "P") frames.Parse++; else if (type === "B") frames.Bind++;
|
||||
else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++;
|
||||
else if (type === "E") frames.Execute++; else if (type === "S") frames.Sync++; else if (type === "X") frames.Terminate++;
|
||||
});
|
||||
observers.add(observer);
|
||||
for (const socket of [client, target]) {
|
||||
|
||||
@@ -45,26 +45,55 @@ describe("assembled maintenance server production wiring", () => {
|
||||
if (!databaseUrl) { temporary = await startEmbeddedPostgresTestDatabase("maintenance-server-"); databaseUrl = temporary.connectionString; }
|
||||
}, 150_000);
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
it("detects legacy aggregate recovery SQL after an empty real server boot settles", async () => {
|
||||
it.each(["empty", "configured terminal history"] as const)("keeps enabled %s quiet for 90 real seconds and two qualified hourly windows", async state => {
|
||||
await clearDatabase();
|
||||
const server = await startMaintenanceServer(databaseUrl!);
|
||||
const fixture = createDb(databaseUrl!, { maxConnections: 1 });
|
||||
if (state !== "empty") {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
|
||||
await fixture.insert(companies).values({ id: companyId, name: "Retained idle history", issuePrefix: "MIH" });
|
||||
await fixture.insert(agents).values({ id: agentId, companyId, name: "Idle retained agent", status: "idle", adapterType: "process",
|
||||
adapterConfig: { command: process.execPath, args: ["-e", "process.exit(0)"] }, runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } });
|
||||
await fixture.insert(issues).values({ id: issueId, companyId, title: "Retained terminal task", status: "done", assigneeAgentId: agentId });
|
||||
await fixture.insert(heartbeatRuns).values({ companyId, agentId, issueId, status: "succeeded", finishedAt: new Date(), contextSnapshot: { issueId } });
|
||||
}
|
||||
const server = await startMaintenanceServer(databaseUrl!, { clock: true });
|
||||
try {
|
||||
const settled = await server.settled();
|
||||
expect(settled.startup.phase).toBe("ready");
|
||||
expect(settled.outgoingHttpRequests).toBe(0);
|
||||
const baseline = server.proxy.queries.length, initialFrames = server.proxy.frames;
|
||||
await delay(31_000);
|
||||
await server.settled();
|
||||
const repeated = server.proxy.queries.slice(baseline);
|
||||
expect(repeated.length).toBeGreaterThan(0);
|
||||
expect(repeated.some(query => query.sql.includes('"heartbeat_runs"'))).toBe(true);
|
||||
if (state !== "empty") await server.api("PATCH", "/instance/settings/experimental", { enableExternalObjects: true, enableStatusCards: true });
|
||||
await settleClock(server); await server.freeze();
|
||||
const before = await server.snapshot(), queryCount = server.proxy.queries.length, frames = server.proxy.frames, transport = server.proxy.transport;
|
||||
expect(before.startup.phase).toBe("ready"); expect(before.idle.active).toBe(0);
|
||||
await server.realtime();
|
||||
const wallStart = Date.now();
|
||||
await delay(90_000);
|
||||
const wallElapsedMs = Date.now() - wallStart;
|
||||
await server.freeze();
|
||||
const real = await server.settled(100);
|
||||
expect(wallElapsedMs).toBeGreaterThanOrEqual(90_000);
|
||||
expect(real.now - before.now).toBeGreaterThanOrEqual(89_000);
|
||||
expect(real.idle.active).toBe(0);
|
||||
expect(server.proxy.queries.length).toBe(queryCount);
|
||||
expect(real.outgoingHttpRequests).toBe(before.outgoingHttpRequests);
|
||||
expect(real.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections);
|
||||
expect(real.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten);
|
||||
const realFrames = server.proxy.frames, realTransport = server.proxy.transport;
|
||||
for (let step = 0; step < 240; step++) { await server.advance(30_000); await server.settled(10); }
|
||||
const after = await server.snapshot();
|
||||
expect(after.now - real.now).toBe(2 * 60 * 60_000);
|
||||
expect(after.idle.active).toBe(0);
|
||||
expect(server.proxy.queries.length).toBe(queryCount);
|
||||
expect(after.outgoingHttpRequests).toBe(before.outgoingHttpRequests);
|
||||
expect(after.network.nonDatabaseConnections).toBe(before.network.nonDatabaseConnections);
|
||||
expect(after.network.nonDatabaseBytesWritten).toBe(before.network.nonDatabaseBytesWritten);
|
||||
expect(server.proxy.errors).toEqual([]);
|
||||
expect((await server.snapshot()).outgoingHttpRequests).toBe(0);
|
||||
await writeFile(path.join(server.directory, "measurement.json"), JSON.stringify({ phase: "legacy-positive-control",
|
||||
observationMs: 31_000, startupQueries: baseline, startup: receipt(server.proxy.queries.slice(0, baseline)),
|
||||
repeatedQueries: repeated.length, repeated: receipt(repeated), initialFrames, finalFrames: server.proxy.frames, outgoingHttpRequests: 0 }, null, 2));
|
||||
} finally { await server.stop(); }
|
||||
}, 150_000);
|
||||
await writeFile(path.join(server.directory, "enabled-quiet-measurement.json"), JSON.stringify({ phase: "enabled-quiet", state,
|
||||
wallElapsedMs, realClockElapsedMs: real.now - before.now, qualifiedClockMs: after.now - real.now, stepMs: 30_000, ioSettledBetweenSteps: true,
|
||||
measuredQueries: 0, startupQueries: queryCount, startup: receipt(server.proxy.queries), before, real, after,
|
||||
frames: { before: frames, real: realFrames, after: server.proxy.frames }, transport: { before: transport, real: realTransport, after: server.proxy.transport },
|
||||
boundary: "actual startServer, real elapsed 90s plus qualified child clock; pool connection housekeeping reported separately; no active provider/native work, Cloud enrollment, installed plugins, telemetry exporter, announcements or database backup; no host or database suspension claim",
|
||||
}, null, 2));
|
||||
} finally { await server.stop(); await fixture.$client.end(); }
|
||||
}, 240_000);
|
||||
|
||||
it("releases an ordinary held task Retry into one real process execution", async () => {
|
||||
await clearDatabase();
|
||||
const server = await startMaintenanceServer(databaseUrl!, { clock: true });
|
||||
|
||||
Reference in new issue
Block a user