From 113e57667e6d13ec20dc5fa110cdb112fb58feae Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:11:53 -0500 Subject: [PATCH] test: distinguish startup readiness from settled maintenance Co-Authored-By: Paperclip --- .../2026-10-30-aggregate-maintenance-sleep.md | 11 ++++- .../helpers/maintenance/tsconfig.json | 2 +- .../maintenance-server-routes.test.ts | 42 ++++++++++++++++++- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md index 58ef7de3f7f..7b8869f60df 100644 --- a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md +++ b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md @@ -87,6 +87,10 @@ the raw/prepared real-PostgreSQL calibration remains. Current assembled cases cover: +- Actual readiness while a fixture-owned PostgreSQL lock holds the real startup + retention INSERT. Health becomes ready with active work and pending SQL; + releasing the lock produces the archive and zero active work. Readiness alone + therefore does not stand in for idle eligibility. - Disabled-heartbeat empty boot across two simulated hourly windows. - Ordinary HTTP company/settings/completed-task creation, independent durable inspection, drain/resume and two additional hourly windows without owner work. @@ -94,6 +98,9 @@ Current assembled cases cover: idle drain starts, followed by its actual commit and external-object source ingestion. Provider work waits during drain, then resume fetches the simulated merged PR, persists terminal state and settles for two hourly windows. +- Actual shutdown during an in-flight provider HTTP read, cancellation and + durable claim release without a false terminal result, followed by a fresh + server reconstructing and completing the retained object. - A real HTTP 503 from the controlled provider, durable retry deadline, no SQL/provider retry during drain, successful resume and quiet after settlement. - Actual Cloud warm standby, signed identity claim and durable canonical identity. @@ -139,7 +146,7 @@ drain, exactly-once resume and existing shutdown/restart behavior. Remaining acceptance before a removal PR: final recovery integration, final recovery first-producer paths through assembled wiring, enabled empty/configured/history -quiet including the real 90-second window, scheduling suppression, startup and -shutdown races, relevant contracts across the combined tree, current-head +quiet including the real 90-second window, startup failure isolation and +remaining final-recovery lifecycle races, relevant contracts across the combined tree, current-head Greptile 5/5 and required CI. No aggregate-removal acceptance or deployment claim is made by the preliminary harness checkpoint. diff --git a/server/src/__tests__/helpers/maintenance/tsconfig.json b/server/src/__tests__/helpers/maintenance/tsconfig.json index 39a4753761d..864a6b58de6 100644 --- a/server/src/__tests__/helpers/maintenance/tsconfig.json +++ b/server/src/__tests__/helpers/maintenance/tsconfig.json @@ -4,5 +4,5 @@ "noEmit": true, "rootDir": "../../../../.." }, - "include": ["*.ts", "../../maintenance-*.test.ts", "../../../types/express.d.ts"] + "include": ["*.ts", "../../maintenance-*.test.ts", "../../decision-target-work.test.ts", "../../../types/express.d.ts"] } diff --git a/server/src/__tests__/maintenance-server-routes.test.ts b/server/src/__tests__/maintenance-server-routes.test.ts index ac5b4e448d2..3a21f5d04fd 100644 --- a/server/src/__tests__/maintenance-server-routes.test.ts +++ b/server/src/__tests__/maintenance-server-routes.test.ts @@ -1,6 +1,6 @@ import { createCipheriv, createHash, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID, sign, verify } from "node:crypto"; import { createServer } from "node:http"; -import { agents, agentApiKeys, agentTaskSessions, authUsers, companies, companyMemberships, connectionGrants, createDb, heartbeatRuns, toolApplications, toolConnections } from "@paperclipai/db"; +import { agents, agentApiKeys, agentTaskSessions, authUsers, companies, companyMemberships, connectionGrants, createDb, heartbeatRuns, issues, issueThreadInteractions, toolApplications, toolConnections } from "@paperclipai/db"; import { access, mkdir, readdir, utimes, writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -65,6 +65,46 @@ describe("assembled maintenance server production wiring", () => { repeatedQueries: repeated.length, repeated: receipt(repeated), initialFrames, finalFrames: server.proxy.frames, outgoingHttpRequests: 0 }, null, 2)); } finally { await server.stop(); } }, 150_000); + it("exposes readiness while real deferred housekeeping is held and settles its work after release", async () => { + await clearDatabase(); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + const companyId = randomUUID(), issueId = randomUUID(), interactionId = randomUUID(); + await fixture.insert(companies).values({ id: companyId, name: "Readiness fixture", issuePrefix: "MRD" }); + await fixture.insert(issues).values({ id: issueId, companyId, title: "Historical human decision", status: "todo" }); + await fixture.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, kind: "request_confirmation", + title: "Historical human choice", continuationPolicy: "none", payload: { version: 1, prompt: "Choose a color?" }, + updatedAt: new Date(Date.now() - 91 * 86_400_000) }); + await fixture.$client.unsafe(`create or replace function maintenance_hold_startup_archive() returns trigger language plpgsql as $$begin + perform pg_advisory_xact_lock(913703); return new; end$$`); + await fixture.$client.unsafe(`create trigger maintenance_hold_startup_archive before insert on decision_retention for each row execute function maintenance_hold_startup_archive()`); + let reserved: Awaited> | undefined = await fixture.$client.reserve(); + await reserved`select pg_advisory_lock(913703)`; + let server: Awaited> | undefined; + try { + server = await startMaintenanceServer(databaseUrl!, { suppressed: true }); + const current = server; + await expect.poll(() => current.proxy.queries.some(query => /insert into "decision_retention"/.test(query.sql)), { timeout: 10_000 }).toBe(true); + const snapshot = await server.snapshot(); + expect(snapshot.startup.phase).toBe("ready"); expect(snapshot.idle.active).toBeGreaterThan(0); + expect(server.proxy.pending).toBeGreaterThan(0); + const health = await server.api<{ status: string; startupRecovery: { phase: string } }>("GET", "/health"); + expect(health).toMatchObject({ status: "ok", startupRecovery: { phase: "ready" } }); + await reserved`select pg_advisory_unlock(913703)`; reserved.release(); reserved = undefined; + await server.settled(); + const [archived] = await fixture.$client`select archived_reason, archived_at is not null as archived from decision_retention where source_id=${interactionId}`; + expect(archived).toMatchObject({ archived_reason: "idle_ttl", archived: true }); + await writeFile(path.join(server.directory, "readiness-measurement.json"), JSON.stringify({ phase: "real-housekeeping-held-after-readiness", + companyId, issueId, interactionId, beforeRelease: { startup: snapshot.startup, idle: snapshot.idle }, afterRelease: await server.snapshot(), + archived, boundary: "fixture owns advisory lock on actual startup archive INSERT; no startup/DB/service mocks", frames: server.proxy.frames, + }, null, 2)); + } finally { + if (reserved) { await reserved`select pg_advisory_unlock(913703)`.catch(() => {}); reserved.release(); } + await fixture.$client.unsafe("drop trigger if exists maintenance_hold_startup_archive on decision_retention").catch(() => {}); + await fixture.$client.unsafe("drop function if exists maintenance_hold_startup_archive()").catch(() => {}); + await fixture.$client.end(); await server?.stop(); + } + }, 150_000); + it("keeps the disabled-heartbeat boot quiet across two former hourly windows with a qualified child clock", async () => { await clearDatabase(); const server = await startMaintenanceServer(databaseUrl!, { enabled: false, clock: true });