PaperClipAI/server/src/__tests__/environment-cleanup-work.test.ts

import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createSandboxOrphanCleanupSpool } from "../services/sandbox-orphan-cleanup-spool.js";
import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
import * as environmentCleanupState from "../services/environment-cleanup-state.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { companies, createDb, sandboxCreationOperations, environmentLeases, environments, type Db } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { registerEnvironmentCleanupWork } from "../services/environment-cleanup-work.js";
import { registerDeferredEnvironmentCleanup } from "../services/environment-cleanup-notifications.js";
import { environmentService } from "../services/environments.js";
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
import { heartbeatService } from "../services/heartbeat.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";

import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";

const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ENVIRONMENT_CLEANUP_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined, db: ReturnType<typeof createDb>;
  let companyId: string, environmentId: string, draining: boolean, failing: boolean;
  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
  let worker: ReturnType<typeof registerEnvironmentCleanupWork> | undefined;
  let releases: ReturnType<typeof vi.fn<() => Promise<void>>>;
  let reads: ReturnType<typeof vi.spyOn>;
  let pendingSweeps: ReturnType<typeof vi.fn<(backoffMs: number) => Promise<unknown>>>;
  let inspections: ReturnType<typeof vi.fn<(row: { providerLeaseId: string | null }) => Promise<{ providerLeaseId: string | null; state: string }>>>;
  const errors: unknown[] = [], disposers: Array<() => void> = [];
  beforeAll(async () => {
    if (!external) database = await startEmbeddedPostgresTestDatabase("environment-cleanup-work-");
    db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
    await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`)));
  }, 90_000);
  beforeEach(async () => {
    await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`);
    companyId = randomUUID(); environmentId = randomUUID(); draining = false; failing = false; errors.length = 0;
    await db.insert(companies).values({ id: companyId, name: "Cleanup worker", issuePrefix: "CLN" });
    await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
    releases = vi.fn(async () => { if (failing) throw new Error("provider unavailable"); return undefined; });
    inspections = vi.fn(async (row: { providerLeaseId: string | null }) => ({ providerLeaseId: row.providerLeaseId, state: "destroyed" }));
    reads = vi.spyOn(db, "select");
    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
  });
  afterEach(async () => {
    worker?.stop(); await coordinator?.stop(); coordinator = undefined; worker = undefined;
    for (const stop of disposers.splice(0)) stop();
    // Creation ownership intentionally survives company deletion; company
    // cascades therefore cannot isolate this case from the next test suite.
    await db.delete(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId));
    vi.useRealTimers(); vi.restoreAllMocks();
    expect(errors).toEqual([]);
  });
  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
  async function start(historical = false, sweepOverride?: (backoffMs: number) => Promise<unknown>, observePendingCheck?: (pending: boolean) => void) {
    const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", retryPendingSandboxTeardown: releases } as any] });
    const heartbeat = heartbeatService(db, { environmentRuntime: runtime });
    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
      onError: error => errors.push(error) });
    if (observePendingCheck) {
      const register = coordinator.register.bind(coordinator);
      vi.spyOn(coordinator, "register").mockImplementation((queue, task) => register(queue, queue !== DELIVERY_QUEUES.environmentCleanup ? task : {
        ...task,
        hasPending() {
          const result = task.hasPending();
          void result.then(value => { observePendingCheck(value); }, () => {});
          return result;
        },
      }));
    }
    const history = historical && historicalCleanupVerificationService(db, { inspectLease: inspections, inspectSetup: inspections });
    let historySettled = !history;
    pendingSweeps = vi.fn(sweepOverride ?? (backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs })));
    worker = registerEnvironmentCleanupWork(db, coordinator, {
      sweepPending: pendingSweeps,
      historical: history ? { hasPending: () => history.hasPending(),
        sweep: async () => { const result = await history.sweep(); historySettled = true; return result; } } : undefined,
      canRun: () => !draining,
    });
    await worker.ready;
    await vi.waitFor(() => expect(historySettled).toBe(true));
    return history || undefined;
  }
  async function settle(ms = 30_000, wait = true) {
    await vi.advanceTimersByTimeAsync(ms);
    if (wait && !draining) {
      await vi.waitFor(() => expect(idleWorkSnapshot().active === 0 || coordinator!.nextWakeAt() !== null).toBe(true), { timeout: 5000 });
      await vi.advanceTimersByTimeAsync(1);
      await vi.waitFor(() => expect(idleWorkSnapshot().active === 0 || coordinator!.nextWakeAt() !== null).toBe(true), { timeout: 5000 });
    }
  }
  const pending = (executor: Db = db) => environmentService(executor).insertPendingCleanupLease({ companyId, environmentId,
    provider: "fixture", providerLeaseId: randomUUID(), metadata: { driver: "sandbox" }, failureReason: "fixture" });
  async function quiet() {
    for (let pass = 0; pass < 4; pass++) {
      await settle(30_000);
      if (coordinator!.nextWakeAt() === null) break;
    }
    expect(coordinator!.nextWakeAt()).toBeNull();
    const count = reads.mock.calls.length, providerCalls = releases.mock.calls.length + inspections.mock.calls.length;
    await settle(20 * 60_000);
    expect(reads).toHaveBeenCalledTimes(count);
    expect(releases.mock.calls.length + inspections.mock.calls.length).toBe(providerCalls);
  }
  it("stays quiet with an idle environment and verified history, and rechecks changed proof", async () => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" });
    await start(true); expect(inspections).toHaveBeenCalledTimes(1);
    await quiet();
    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", changed: true });
    await settle(); expect(inspections).toHaveBeenCalledTimes(2);
    await environmentService(db).updateLeaseMetadata(lease.id, { changed: "again" });
    await settle(90_000); expect(inspections).toHaveBeenCalledTimes(2);
    await settle(180_000); expect(inspections).toHaveBeenCalledTimes(3);
    await quiet();
  });
  it("reconstructs a creation deadline after restart and leaves manual reconciliation quiet", async () => {
    await start(); await quiet();
    await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
      async () => ({ providerLeaseId: "fixture-unattached" }));
    await settle();
    expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
    worker!.stop(); await coordinator!.stop();
    await start();
    expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now());
    await settle(31 * 60_000);
    await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
      expect.objectContaining({ state: "needs_reconciliation" }),
    ]));
    await quiet();
  });
  it.each([false, true])("retains deleted-company creation uncertainty across a quiet worker restart (contextual=%s)", async contextual => {
    await start(); await quiet();
    const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
      metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
      async () => ({ providerLeaseId: "fixture-deleted-company" }));
    await db.delete(companies).where(eq(companies.id, companyId));
    await settle(31 * 60_000);
    await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
      expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
    ]));
    if (contextual) {
      if (!created.providerLeaseId) throw new Error("Fixture creation returned no provider lease identity");
      await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
        providerLeaseId: created.providerLeaseId });
    }
    await quiet();
    worker!.stop(); await coordinator!.stop();
    await start(); await quiet();
    expect(releases).not.toHaveBeenCalled(); expect(inspections).not.toHaveBeenCalled();
    const ownerId = randomUUID();
    const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
    expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
      .toEqual({ version: 1, backgroundWork: "present" });
    await quiet();
  });
  it("discharges an empty pending cleanup metadata notification during provider spacing", async () => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
    const startedAt = Date.now();
    await start();
    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
    expect(idleWorkSnapshot().active).toBe(0);
    expect(coordinator!.nextWakeAt()).toBeNull();
    expect(pendingSweeps).toHaveBeenCalledTimes(1);
    expect(Date.now() - startedAt).toBeLessThan(30_000);
    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", presentation: "changed" });
    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
    await vi.waitFor(() => {
      expect(idleWorkSnapshot().active).toBe(0);
      expect(coordinator!.nextWakeAt()).toBeNull();
    });
    expect(Date.now() - startedAt).toBeLessThan(30_000);
    expect(pendingSweeps).toHaveBeenCalledTimes(1);
    expect(releases).not.toHaveBeenCalled();
    const executions = vi.spyOn(db, "execute");
    const count = reads.mock.calls.length;
    await vi.advanceTimersByTimeAsync(20 * 60_000);
    expect(reads).toHaveBeenCalledTimes(count);
    expect(executions).not.toHaveBeenCalled();
    expect(pendingSweeps).toHaveBeenCalledTimes(1);
    expect(releases).not.toHaveBeenCalled();
  });
  it.each([false, true])("preserves pending cleanup debt arriving during an empty fast-path read (unresolved transaction=%s)", async unresolved => {
    const passive = await pending();
    await environmentService(db).releaseLease(passive.id, "released", { cleanupStatus: "success" });
    await start();
    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
    let readStarted = false, releaseRead!: () => void, writeReady!: () => void, releaseWrite!: () => void;
    const readGate = new Promise<void>(resolve => { releaseRead = resolve; });
    const written = new Promise<void>(resolve => { writeReady = resolve; });
    const writeGate = new Promise<void>(resolve => { releaseWrite = resolve; });
    const original = environmentCleanupState.nextEnvironmentCleanupAt;
    const check = vi.spyOn(environmentCleanupState, "nextEnvironmentCleanupAt").mockImplementationOnce(async (...args) => {
      const at = await original(...args); expect(at).toBeNull(); readStarted = true; await readGate; return at;
    });
    let writing: Promise<unknown> | undefined, debtId: string | undefined;
    try {
      await environmentService(db).updateLeaseMetadata(passive.id, { presentation: "changed" });
      await vi.waitFor(() => expect(readStarted).toBe(true));
      writing = db.transaction(async tx => {
        debtId = (await pending(tx as unknown as Db)).id;
        writeReady(); if (unresolved) await writeGate;
      });
      await written;
      if (!unresolved) await writing;
      releaseRead();
      await vi.advanceTimersByTimeAsync(1);
      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
      expect(pendingSweeps).toHaveBeenCalledTimes(1);
      expect(releases).not.toHaveBeenCalled();
      releaseWrite(); await writing;
      await vi.waitFor(async () => expect(await original(db)).not.toBeNull());
      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
      const ownerId = randomUUID();
      const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000),
        expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
      expect((await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")).backgroundWork).toBe("present");
      await settle(300_000);
      await vi.waitFor(() => { expect(releases).toHaveBeenCalledTimes(1); expect(idleWorkSnapshot().active).toBe(0); });
      expect((await environmentService(db).getLeaseById(debtId!))?.status).toBe("expired");
      expect(await original(db)).toBeNull(); await quiet();
    } finally { releaseRead(); releaseWrite(); await writing; check.mockRestore(); }
  });
  it("retains a failed pending cleanup fast-path read through drain and clears it after resume", async () => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
    await start();
    expect(await nextEnvironmentCleanupAt(db)).toBeNull();
    const original = environmentCleanupState.nextEnvironmentCleanupAt;
    vi.spyOn(environmentCleanupState, "nextEnvironmentCleanupAt").mockImplementationOnce(async (...args) => {
      expect(await original(...args)).toBeNull();
      throw new Error("pending cleanup deadline read unavailable");
    });
    await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "changed" });
    await vi.waitFor(() => expect(errors).toHaveLength(1));
    expect(idleWorkSnapshot().active).toBeGreaterThan(0); expect(coordinator!.nextWakeAt()).not.toBeNull();
    expect(pendingSweeps).toHaveBeenCalledTimes(1); expect(releases).not.toHaveBeenCalled();
    errors.length = 0; draining = true;
    const count = reads.mock.calls.length;
    await settle(60_000, false);
    expect(reads).toHaveBeenCalledTimes(count); expect(idleWorkSnapshot().active).toBeGreaterThan(0);
    expect(pendingSweeps).toHaveBeenCalledTimes(1);
    draining = false; await settle();
    await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
    expect(releases).not.toHaveBeenCalled(); await quiet();
    worker!.stop(); await coordinator!.stop();
    expect(idleWorkSnapshot().active).toBe(0);
  });
  it("discharges an empty historical metadata notification during provider spacing", async () => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
    const history = await start(true);
    await settle(30_000);
    expect(await history!.hasPending()).toBe(false);
    const checked = inspections.mock.calls.length;
    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", presentation: "changed" });
    await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
    expect(inspections).toHaveBeenCalledTimes(checked);
    expect(await history!.hasPending()).toBe(false);
    const count = reads.mock.calls.length;
    await vi.advanceTimersByTimeAsync(300_000);
    expect(reads).toHaveBeenCalledTimes(count); expect(inspections).toHaveBeenCalledTimes(checked);
  });
  it.each([false, true])("preserves historical debt arriving during an empty fast-path read (unresolved transaction=%s)", async unresolved => {
    const passive = await pending(), debt = await pending();
    await environmentService(db).releaseLease(passive.id, "released", { cleanupStatus: "success" });
    await environmentService(db).releaseLease(debt.id, "failed", { cleanupStatus: "success" });
    const history = (await start(true))!;
    await settle(30_000); expect(inspections).toHaveBeenCalledTimes(1);
    let readStarted = false, releaseRead!: () => void, writeReady!: () => void, releaseWrite!: () => void;
    const readGate = new Promise<void>(resolve => { releaseRead = resolve; });
    const written = new Promise<void>(resolve => { writeReady = resolve; });
    const writeGate = new Promise<void>(resolve => { releaseWrite = resolve; });
    const original = history.hasPending;
    const check = vi.spyOn(history, "hasPending").mockImplementationOnce(async () => {
      const pending = await original(); expect(pending).toBe(false); readStarted = true; await readGate; return pending;
    });
    let writing: Promise<unknown> | undefined;
    try {
      await environmentService(db).updateLeaseMetadata(passive.id, { presentation: "changed" });
      await vi.waitFor(() => expect(readStarted).toBe(true));
      writing = db.transaction(async tx => {
        await environmentService(tx as unknown as Db).updateLeaseMetadata(debt.id, { driver: "sandbox", changed: true });
        writeReady(); if (unresolved) await writeGate;
      });
      await written;
      if (!unresolved) await writing;
      releaseRead();
      await vi.advanceTimersByTimeAsync(1);
      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
      expect(inspections).toHaveBeenCalledTimes(1);
      releaseWrite(); await writing;
      await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1); expect(await original()).toBe(true); });
      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
      const ownerId = randomUUID();
      // One drain hold must retain its identity while the safety read runs.
      const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000),
        expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
      expect((await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")).backgroundWork).toBe("present");
      await settle(300_000);
      await vi.waitFor(() => { expect(inspections).toHaveBeenCalledTimes(2); expect(idleWorkSnapshot().active).toBe(0); });
      expect(await original()).toBe(false); await quiet();
    } finally { releaseRead(); releaseWrite(); await writing; check.mockRestore(); }
  });
  it("retains a failed historical fast-path read through drain and clears it after resume", async () => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
    const history = (await start(true))!;
    await settle(30_000);
    vi.spyOn(history, "hasPending").mockRejectedValueOnce(new Error("historical debt read unavailable"));
    await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "changed" });
    await vi.waitFor(() => expect(errors).toHaveLength(1));
    expect(idleWorkSnapshot().active).toBeGreaterThan(0); expect(coordinator!.nextWakeAt()).not.toBeNull();
    errors.length = 0; draining = true;
    const count = reads.mock.calls.length;
    await settle(60_000, false);
    expect(reads).toHaveBeenCalledTimes(count); expect(idleWorkSnapshot().active).toBeGreaterThan(0);
    draining = false; await settle();
    await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
    expect(inspections).not.toHaveBeenCalled(); await quiet();
    worker!.stop(); await coordinator!.stop();
    expect(idleWorkSnapshot().active).toBe(0);
  });
  it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => {
    const lease = await pending();
    await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", creationOperationId: randomUUID() });
    await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" });
    await start(true);
    expect(inspections).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: lease.providerLeaseId,
      metadata: expect.objectContaining({ creationOperationId: expect.any(String) }) }));
    await quiet();
    worker!.stop(); await coordinator!.stop();
    await start(true); expect(inspections).toHaveBeenCalledTimes(1); await quiet();
  });
  it("preserves a first creation signal during drain and disarms after atomic attachment", async () => {
    await start(); await quiet(); draining = true;
    const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
      async () => ({ providerLeaseId: "fixture-attached" }));
    const count = reads.mock.calls.length;
    await settle(60_000, false); expect(reads).toHaveBeenCalledTimes(count);
    draining = false;
    await environmentService(db).acquireLease({ companyId, environmentId, provider: "fixture", providerLeaseId: created.providerLeaseId,
      creationOperationId: String(created.metadata!.creationOperationId), metadata: created.metadata });
    await settle();
    expect(await db.select().from(sandboxCreationOperations)).toEqual([expect.objectContaining({ state: "attached" })]);
    await quiet();
  });
  it("wakes on a real orphan insert, waits for its backoff, then disarms after deletion", async () => {
    await start(); await quiet();
    const lease = await pending(); await settle();
    expect(releases).not.toHaveBeenCalled();
    const count = reads.mock.calls.length;
    await settle(240_000); expect(reads).toHaveBeenCalledTimes(count);
    await settle(30_000); expect(releases).toHaveBeenCalledTimes(1);
    expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired");
    await quiet();
  });
  it("does not rescan pending cleanup from an otherwise idle run-reaper pass", async () => {
    const lease = await pending();
    await db.update(environmentLeases).set({ updatedAt: new Date(Date.now() - 600_000) }).where(eq(environmentLeases.id, lease.id));
    const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", retryPendingSandboxTeardown: releases } as any] });
    await heartbeatService(db, { environmentRuntime: runtime }).reapOrphanedRuns({ staleThresholdMs: 300_000 });
    expect(releases).not.toHaveBeenCalled();
    await start(); expect(releases).toHaveBeenCalledTimes(1); await quiet();
  });
  it("retries a concrete failed teardown and reconstructs its cooldown after restart", async () => {
    const lease = await pending(); failing = true;
    await start(); expect(releases).toHaveBeenCalledTimes(1);
    const due = await nextEnvironmentCleanupAt(db); expect(due).toBeGreaterThan(Date.now());
    worker!.stop(); await coordinator!.stop();
    await start(); expect(releases).toHaveBeenCalledTimes(1);
    failing = false; await settle(300_000); expect(releases).toHaveBeenCalledTimes(2);
    expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired");
    await quiet();
  });
  it.each([false, true])("respects outer transaction settlement (rollback %s)", async rollback => {
    await start(); await quiet();
    let release!: () => void, written!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    const ready = new Promise<void>(resolve => { written = resolve; });
    const writing = db.transaction(async tx => { await pending(tx as unknown as Db); written(); await gate; if (rollback) throw new Error("rollback"); })
      .catch(error => { if (!rollback) throw error; });
    await ready; await settle(30_000, false); expect(releases).not.toHaveBeenCalled();
    release(); await writing; await settle(300_000);
    expect(releases).toHaveBeenCalledTimes(rollback ? 0 : 1); await quiet();
  });
  it("preserves the first write during drain without passive SQL", async () => {
    await start(); await quiet(); draining = true;
    await pending(); const count = reads.mock.calls.length;
    await settle(600_000); expect(reads).toHaveBeenCalledTimes(count); expect(releases).not.toHaveBeenCalled();
    draining = false; await settle(); expect(releases).toHaveBeenCalledTimes(1); await quiet();
  });
  it("wakes for deferred work from another runtime and retries while no durable row exists", async () => {
    await start(); await quiet();
    let available = false, stop!: () => void;
    const flush = vi.fn(async () => {
      if (!available) return { recovered: 0, pending: 1 };
      await pending(); stop(); return { recovered: 1, pending: 0 };
    });
    stop = registerDeferredEnvironmentCleanup(db, flush); disposers.push(stop);
    await settle(); expect(flush).toHaveBeenCalledTimes(1);
    expect(await db.select().from(environmentLeases)).toEqual([]);
    const count = reads.mock.calls.length;
    await settle(240_000); expect(reads).toHaveBeenCalledTimes(count);
    available = true; await settle(60_000); expect(flush).toHaveBeenCalledTimes(2);
    await settle(300_000); expect(releases).toHaveBeenCalledTimes(1); await quiet();
  });
  it("retries a failed spool read without starving other buffers or persisted cleanup", async () => {
    const lease = await pending();
    const stop = registerDeferredEnvironmentCleanup(db, async () => { throw new Error("spool unavailable"); });
    disposers.push(stop);
    let stopHealthy!: () => void;
    let bufferedLeaseId: string | undefined;
    const healthy = vi.fn(async () => {
      bufferedLeaseId = (await pending()).id;
      stopHealthy();
      return { recovered: 1, pending: 0 };
    });
    stopHealthy = registerDeferredEnvironmentCleanup(db, healthy); disposers.push(stopHealthy);
    await start();
    expect(healthy).toHaveBeenCalledTimes(1);
    expect(releases).toHaveBeenCalledTimes(2);
    expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired");
    expect((await environmentService(db).getLeaseById(bufferedLeaseId!))?.status).toBe("expired");
    expect(errors).toHaveLength(1); errors.length = 0;
    draining = true; const count = reads.mock.calls.length;
    await settle(90_000); expect(reads).toHaveBeenCalledTimes(count);
    stop(); draining = false; await quiet();
  });
  it("retains startup recovery for an unread real spool with no buffered or durable row", async () => {
    const dir = await mkdtemp(join(tmpdir(), "cleanup-unread-spool-"));
    try {
      await writeFile(join(dir, "unread.json"), "{ truncated");
      const runtime = environmentRuntimeService(db, { orphanCleanupSpool: createSandboxOrphanCleanupSpool(dir) });
      const heartbeat = heartbeatService(db, { environmentRuntime: runtime });
      await start(false, backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs }));
      expect(errors).toHaveLength(1); errors.length = 0;
      expect(coordinator!.nextWakeAt()).not.toBeNull();
      expect(await db.select().from(environmentLeases)).toEqual([]);
      await rm(join(dir, "unread.json"));
      await quiet();
    } finally { await rm(dir, { recursive: true, force: true }); }
  });
  it.each(["startup", "later"] as const)("retains a failed %s spool sweep through empty metadata during spacing", async phase => {
    const lease = await pending();
    await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" });
    const dir = await mkdtemp(join(tmpdir(), "cleanup-failed-sweep-"));
    const unread = join(dir, "unread.json");
    const spool = createSandboxOrphanCleanupSpool(dir);
    const checks: boolean[] = [];
    const providerLeaseId = randomUUID();
    try {
      if (phase === "startup") await writeFile(unread, "{ truncated");
      const runtime = environmentRuntimeService(db, { orphanCleanupSpool: spool });
      vi.spyOn(runtime, "retryPendingSandboxTeardown").mockImplementation(async () => releases());
      const heartbeat = heartbeatService(db, { environmentRuntime: runtime });
      await start(false, backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs }), value => { checks.push(value); });
      if (phase === "later") {
        expect(errors).toEqual([]); expect(pendingSweeps).toHaveBeenCalledTimes(1);
        expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull();
        await vi.advanceTimersByTimeAsync(30_000);
        await writeFile(unread, "{ truncated");
        await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "request later sweep" });
      }
      await vi.waitFor(() => expect(errors).toHaveLength(1));
      const sweeps = phase === "startup" ? 1 : 2;
      expect(pendingSweeps).toHaveBeenCalledTimes(sweeps);
      expect(await nextEnvironmentCleanupAt(db)).toBeNull();
      expect(coordinator!.nextWakeAt()).not.toBeNull();
      expect(releases).not.toHaveBeenCalled();
      errors.length = 0;
      const failedAt = Date.now();
      const before = checks.length;
      await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "empty hint after failed sweep" });
      expect(await nextEnvironmentCleanupAt(db)).toBeNull();
      await vi.waitFor(() => expect(checks.length).toBeGreaterThan(before));
      expect(checks.at(-1)).toBe(true);
      expect(idleWorkSnapshot().active).toBeGreaterThan(0);
      expect(coordinator!.nextWakeAt()).not.toBeNull();
      expect(Date.now() - failedAt).toBeLessThan(30_000);
      expect(pendingSweeps).toHaveBeenCalledTimes(sweeps);
      expect(await spool.append({ companyId, environmentId, executionWorkspaceId: null, issueId: null,
        heartbeatRunId: null, provider: "fixture", providerLeaseId, metadata: { driver: "sandbox" } })).toBe(true);
      await rm(unread);
      await settle(30_000);
      await vi.waitFor(async () => {
        const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
        expect(rows).toHaveLength(1);
      });
      if (phase === "later") {
        expect(releases).not.toHaveBeenCalled();
        await settle(300_000);
      }
      await vi.waitFor(() => expect(releases).toHaveBeenCalledTimes(1));
      const [recovered] = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
      expect(recovered.status).toBe("expired");
      expect(await spool.loadWithStatus!()).toEqual({ records: [], readFailed: false });
      await quiet();
    } finally { await rm(dir, { recursive: true, force: true }); }
  });
  it("does not delay startup for historical provider checks but joins them on shutdown", async () => {
    let release!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    const sweep = vi.fn(() => gate);
    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
    worker = registerEnvironmentCleanupWork(db, coordinator, {
      sweepPending: async () => {}, historical: { sweep, hasPending: async () => false }, canRun: () => true,
    });
    let ready = false, stopped = false;
    void worker.ready.then(() => { ready = true; });
    try {
      await vi.waitFor(() => expect(ready).toBe(true));
      expect(sweep).toHaveBeenCalledOnce();
      const stopping = coordinator.stop().then(() => { stopped = true; });
      await vi.advanceTimersByTimeAsync(1); expect(stopped).toBe(false);
      release(); await stopping; expect(stopped).toBe(true);
    } finally { release(); }
  });
  it("coalesces a write during a slow sweep and waits for it on shutdown", async () => {
    await start(); await quiet(); worker!.stop(); await coordinator!.stop();
    let release!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    let calls = 0;
    const starting = start(false, async () => { calls++; if (calls === 1) await gate; });
    await settle(1, false); await pending();
    let stopped = false; const stopping = coordinator!.stop().then(() => { stopped = true; });
    await settle(30_000, false); expect(stopped).toBe(false); expect(calls).toBe(1);
    release(); await starting; await stopping; expect(stopped).toBe(true);
  });
  it("rotates historical pages until all confirmed records are passive", async () => {
    for (let i = 0; i < 33; i++) {
      const lease = await pending(); await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" });
    }
    await start(true);
    await vi.waitFor(async () => {
      await settle(300_000);
      expect(inspections).toHaveBeenCalledTimes(33);
    }, { timeout: 5000 });
    await quiet();
  });
  it("keeps unknown historical resources scheduled until provider confirmation", async () => {
    const lease = await pending(); await environmentService(db).releaseLease(lease.id, "failed", { cleanupStatus: "success" });
    inspections.mockResolvedValueOnce({ providerLeaseId: lease.providerLeaseId, state: "unknown" });
    await start(true);
    // The provider sweep finishes before its pending-state read registers the
    // retry. Readiness intentionally does not wait for historical verification.
    await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()));
    await settle(300_000); expect(inspections).toHaveBeenCalledTimes(2); await quiet();
  });
});