diff --git a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md index 7b8869f60df..ca4407bd2b2 100644 --- a/doc/plans/2026-10-30-aggregate-maintenance-sleep.md +++ b/doc/plans/2026-10-30-aggregate-maintenance-sleep.md @@ -87,11 +87,21 @@ the raw/prepared real-PostgreSQL calibration remains. Current assembled cases cover: +- A fixture-only goal-control INSERT error that previously aborted actual boot. + With owner isolation, the goal row stays pending, a new real residual lease + obligation is discovered within one second of child time and is cleaned by its + separate owner, and ordinary attention ingestion still archives its source. + Native classification remains fail-closed. - Actual readiness while a fixture-owned PostgreSQL lock holds the real startup retention INSERT. Health becomes ready with active work and pending SQL; releasing the lock produces the archive and zero active work. Readiness alone therefore does not stand in for idle eligibility. - Disabled-heartbeat empty boot across two simulated hourly windows. +- Actual worktree suppression with a committed residual obligation and no + recurring SQL over two minutes of child time. A normal operator settings + activation reconstructs that obligation and produces a durable cleanup result. + Activation uses the stored instance identity and timestamp; no direct worker + wake or safety-gate bypass is used. - Ordinary HTTP company/settings/completed-task creation, independent durable inspection, drain/resume and two additional hourly windows without owner work. - An ordinary HTTP issue insertion held by an isolated PostgreSQL lock while diff --git a/server/src/__tests__/maintenance-server-routes.test.ts b/server/src/__tests__/maintenance-server-routes.test.ts index 976481dc53b..39680b0d36b 100644 --- a/server/src/__tests__/maintenance-server-routes.test.ts +++ b/server/src/__tests__/maintenance-server-routes.test.ts @@ -65,6 +65,39 @@ describe("assembled maintenance server production wiring", () => { repeatedQueries: repeated.length, repeated: receipt(repeated), initialFrames, finalFrames: server.proxy.frames, outgoingHttpRequests: 0 }, null, 2)); } finally { await server.stop(); } }, 150_000); + it("reconstructs residual work from a normal scheduler-admission change after suppression", async () => { + await clearDatabase(); + const server = await startMaintenanceServer(databaseUrl!, { clock: true, suppressed: true }); + const fixture = createDb(databaseUrl!, { maxConnections: 1 }); + try { + await server.settled(); + const company = await server.api<{ id: string }>("POST", "/companies", { name: "Admission reconstruction" }); + const { leaseId } = await server.createOrphanLease(company.id); + await server.advance(1_000); await settleClock(server); + const [suppressed] = await fixture.$client`select status, cleanup_status from environment_leases where id=${leaseId}`; + expect(suppressed).toMatchObject({ status: "active", cleanup_status: null }); + const baseline = server.proxy.queries.length, before = await server.snapshot(); + for (let step = 0; step < 4; step++) { await server.advance(30_000); await server.settled(10); } + expect(server.proxy.queries.length).toBe(baseline); + expect((await server.snapshot()).outgoingHttpRequests).toBe(before.outgoingHttpRequests); + await server.api("PATCH", "/instance/settings/experimental", { enableWorktreeRunExecution: true }); + await server.advance(1_000); await settleClock(server); + const [admitted] = await fixture.$client`select experimental from instance_settings where singleton_key='default'`; + expect(admitted!.experimental).toMatchObject({ enableWorktreeRunExecution: true, + worktreeRunExecutionActivatedAt: expect.any(String), worktreeRunExecutionActivationInstanceId: "maintenance-test" }); + const [reconstructed] = await fixture.$client`select status, failure_reason, cleanup_status from environment_leases where id=${leaseId}`; + expect(reconstructed).toMatchObject({ status: "expired", failure_reason: "pending_cleanup_retry", cleanup_status: "success" }); + await server.advance(30_000); await settleClock(server); + const [cleaned] = await fixture.$client`select status, cleanup_status from environment_leases where id=${leaseId}`; + expect(cleaned).toMatchObject({ status: "expired", cleanup_status: "success" }); + await writeFile(path.join(server.directory, "admission-measurement.json"), JSON.stringify({ phase: "ordinary-settings-admission-reconstruction", + companyId: company.id, leaseId, suppressed, suppressedClockMs: 120_000, suppressedQueries: 0, + admitted: admitted!.experimental, reconstructed, cleaned, frames: server.proxy.frames, + boundary: "actual suppressed runtime, committed environment-cleanup ingress, normal operator activation; no direct worker wake or safety-gate bypass", + }, null, 2)); + } finally { await server.stop(); await fixture.$client.end(); } + }, 150_000); + it("isolates a real goal-startup write failure and still registers residual and attention owners", async () => { await clearDatabase(); const fixture = createDb(databaseUrl!, { maxConnections: 1 }); @@ -671,12 +704,26 @@ describe("assembled maintenance server production wiring", () => { const [projected] = await fixture.$client`select rr.company_id, rr.status, rr.completed_at is not null as completed, i.status as issue_status from routine_runs rr join issues i on i.id=rr.linked_issue_id where rr.id=${run.id}`; expect(projected).toMatchObject({ company_id: companyId, status: "completed", completed: true, issue_status: "done" }); + const restoredRun = await server.api<{ id: string; linkedIssueId: string }>("POST", `/routines/${routine.id}/run`, { + source: "manual", idempotencyKey: randomUUID(), + }); + const cancel = await server.api<{ hold: { id: string } }>("POST", `/issues/${restoredRun.linkedIssueId}/tree-holds`, { mode: "cancel", reason: "Ordinary cancel before restore" }); + await server.advance(1_000); await settleClock(server); + const [cancelled] = await fixture.$client`select status from routine_runs where id=${restoredRun.id}`; + expect(cancelled!.status).toBe("failed"); + await server.api("POST", `/issues/${restoredRun.linkedIssueId}/tree-holds`, { mode: "restore", reason: "Ordinary restore" }); + await server.advance(1_000); await settleClock(server); + const [restored] = await fixture.$client`select rr.status, rr.completed_at, rr.failure_reason, i.status as issue_status + from routine_runs rr join issues i on i.id=rr.linked_issue_id where rr.id=${restoredRun.id}`; + expect(restored).toMatchObject({ status: "issue_created", completed_at: null, failure_reason: null, issue_status: "todo" }); + const [cancelHold] = await fixture.$client`select status from issue_tree_holds where id=${cancel.hold.id}`; + expect(cancelHold!.status).toBe("released"); const [executions] = await fixture.$client`select count(*)::integer as count from heartbeat_runs where company_id=${companyId}`; expect(executions!.count).toBe(0); expect(storageRequests).toEqual([]); await writeFile(path.join(server.directory, "routine-measurement.json"), JSON.stringify({ phase: "ordinary-routine-service-completion-during-drain", completionBoundary: "actual issueService.update with runtime-owned DB; excludes redundant route projection hook", companyId, routineId: routine.id, runId: run.id, - before, drained, projected, executions: executions!.count, storageRequests, + before, drained, projected, restoredRun, cancelled, restored, cancelHold, executions: executions!.count, storageRequests, suppression: "PAPERCLIP_IN_WORKTREE; actual production policy", frames: server.proxy.frames, }, null, 2)); } finally {