PaperClipAI/server/src/__tests__/idle-domain-deadlines-postgres.test.ts

import { createGitHubGrantContinuity, readIdleGitHubGrantContinuity } from "../services/tool-access/github-grant-continuity.js";
import { createOAuthCredentialStore } from "../services/tool-access/oauth/credential-store.js";
import { createOAuthRefreshLeases } from "../services/tool-access/oauth/refresh-leases.js";
import { createOAuthGrantRefresh } from "../services/tool-access/oauth/grant-refresh.js";
import { secretService } from "../services/secrets.js";
import { writeConnectionCredential } from "../services/connection-credentials.js";
import type { PaperclipCloudConnector } from "../services/paperclip-cloud-connector.js";
import { GITHUB_CONNECTOR_PROFILES } from "@paperclipai/shared";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { createRequire } from "node:module";
import { setTimeout as delay } from "node:timers/promises";
import { createPluginWorkerManager, type PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { createPluginJobScheduler } from "../services/plugin-job-scheduler.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { pluginStateStore } from "../services/plugin-state-store.js";
import { startTaskDrain, stopTaskDrain, beginIdleTrackedWork } from "../services/task-admission.js";
import { providerTraceStore } from "../services/provider-trace-store.js";
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
import { randomUUID } from "node:crypto";
import { eq, sql, inArray } from "drizzle-orm";
import { agents, companies, heartbeatRuns, plugins, pluginJobs, pluginJobRuns, pluginState, providerTraceRecords, issues, issueWatchdogs, issueThreadInteractions, toolApplications, toolConnections, connectionGrants, companySecrets, companySecretVersions, routines, routineTriggers,
  applyPendingMigrations, createDb, type Db } from "@paperclipai/db";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { readIdleDeadlineProof } from "../services/idle-deadline-proof.js";
import type { VerifiedIdleCapabilities } from "../services/idle-incoming-wake-capability.js";
import { claimDueTimerHeartbeatForAgent } from "../services/heartbeat/queue.js";
import { createHeartbeatTimerPolicy } from "../services/heartbeat/timer-policy.js";
import { pluginJobStore } from "../services/plugin-job-store.js";

const external = resolveExternalTestDatabaseUrl("PAPERCLIP_IDLE_DEADLINE_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("durable domain deadline snapshot with PostgreSQL", () => {
  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
  let db: Db;
  let directory: string;
  beforeAll(async () => {
    if (external) await applyPendingMigrations(external);
    else temporary = await startEmbeddedPostgresTestDatabase("idle-domain-deadlines-");
    db = createDb(external ?? temporary!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
    directory = await fs.mkdtemp(path.join(os.tmpdir(), "idle-deadline-producers-"));
    vi.stubEnv("PAPERCLIP_CONFIG", path.join(directory, "config.json"));
    vi.stubEnv("PROVIDER_TRACE_BASE_PATH", path.join(directory, "traces"));
    vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk");
    vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "d".repeat(64));
  }, 60_000);
  afterAll(async () => { vi.useRealTimers(); vi.unstubAllEnvs(); await db?.$client.end({ timeout: 1 }); await temporary?.cleanup();
    if (directory) await fs.rm(directory, { recursive: true, force: true }); });

  it.each(["routines", "issues"] as const)("marks overflowed %s incomplete under the negotiated deadline catalog", async family => {
    const now = Date.now(), companyId = randomUUID(), agentId = randomUUID(), ownerId = randomUUID();
    const capability = { deadlineCatalog: "domain_deadlines_v1" } as VerifiedIdleCapabilities;
    const read = () => readIdleSleepSafety(db, () => ({ ownerId, draining: true,
      startedAt: new Date(now - 1000), expiresAt: new Date(now + 300_000), activeRuns: 0, pendingWakes: 0 }),
      () => now, ownerId, async () => "none", undefined, undefined, true, true,
      { deadlineCapability: capability, collectDiagnostics: true });
    const reason = `${family}.schedule`;
    const ids = Array.from({ length: 10_001 }, () => randomUUID());
    const insert = async (batch: string[]) => {
      if (family === "routines") {
        await db.insert(routines).values(batch.map(id => ({ id, companyId, title: "Future routine",
          status: "active", assigneeAgentId: agentId })));
        await db.insert(routineTriggers).values(batch.map(id => ({ id, routineId: id, companyId,
          kind: "schedule", cronExpression: "*/15 * * * *", timezone: "UTC", nextRunAt: new Date(now + 600_000),
          enabled: true, archived: false, setupPending: false })));
      } else {
        await db.insert(issues).values(batch.map(id => ({ id, companyId, title: "Passive backlog", status: "backlog" })));
      }
    };
    await db.insert(companies).values({ id: companyId, name: "Deadline overflow", issuePrefix: `O${companyId.slice(0, 6)}` });
    try {
      await db.insert(agents).values({ id: agentId, companyId, name: "On-demand worker", role: "engineer",
        status: "idle", adapterType: "process" });
      for (let offset = 0; offset < 10_000; offset += 500) await insert(ids.slice(offset, Math.min(offset + 500, 10_000)));
      const atLimit = await read();
      expect(atLimit).toMatchObject({ version: 3, backgroundWork: "none", diagnostics: { complete: true },
        deadlineProof: { catalog: "domain_deadlines_v1" } });
      expect(atLimit.diagnostics?.checks).toContainEqual({ reason, state: "none" });
      if (family === "routines") expect(atLimit.scheduledWake?.nextWakeAt).toBe(new Date(now + 600_000).toISOString());
      else expect(atLimit.scheduledWake).toBeUndefined();

      await insert([ids[10_000]!]);
      const [rowCount] = await db.select({ count: sql<number>`count(*)::integer` })
        .from(family === "routines" ? routines : issues).where(eq(family === "routines" ? routines.companyId : issues.companyId, companyId));
      expect(rowCount?.count).toBe(10_001);
      if (family === "routines") {
        const [triggers] = await db.select({ count: sql<number>`count(*)::integer` }).from(routineTriggers)
          .where(eq(routineTriggers.companyId, companyId));
        expect(triggers?.count).toBe(10_001);
      }
      const overflowed = await read();
      expect(overflowed.backgroundWork).not.toBe("none");
      expect(overflowed.diagnostics?.complete).toBe(false);
      expect(overflowed.diagnostics?.checks).toContainEqual({ reason, state: "unknown" });
      expect(overflowed.deadlineProof).toBeUndefined();
      expect(overflowed.scheduledWake).toBeUndefined();

      if (family === "routines") {
        await db.delete(routines).where(eq(routines.id, ids[10_000]!));
        await db.update(routineTriggers).set({ nextRunAt: new Date(now - 1) }).where(eq(routineTriggers.id, ids[0]!));
      } else {
        await db.delete(issues).where(eq(issues.id, ids[10_000]!));
        await db.update(issues).set({ status: "in_progress", assigneeAgentId: agentId }).where(eq(issues.id, ids[0]!));
      }
      const blocked = await read();
      expect(blocked.backgroundWork).not.toBe("none");
      expect(blocked.diagnostics?.complete).toBe(true);
      expect(blocked.diagnostics?.checks).toContainEqual({ reason, state: "present" });
      expect(blocked.deadlineProof).toBeUndefined();
      expect(blocked.scheduledWake).toBeUndefined();
    } finally {
      await db.delete(routines).where(eq(routines.companyId, companyId));
      await db.delete(issues).where(eq(issues.companyId, companyId));
      await db.delete(agents).where(eq(agents.companyId, companyId));
      await db.delete(companies).where(eq(companies.id, companyId));
    }
  }, 30_000);

  it.each([
    ["2099-01-01T12:00:00.000Z", "2099-01-02T00:00:00.000Z"],
    ["2099-01-01T23:59:06.888Z", "2099-01-02T00:06:36.888Z"],
    ["2099-01-02T00:00:00.500Z", "2099-01-03T00:00:00.000Z"],
  ])("preserves both the timer interval and UTC daily cap at %s", async (instant, deadline) => {
    const now = new Date(instant), companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
    await db.insert(companies).values({ id: companyId, name: "UTC timer fixture", issuePrefix: `T${companyId.slice(0, 6)}` });
    try {
      const [agent] = await db.insert(agents).values({ id: agentId, companyId, name: "Capped timer", role: "engineer",
        status: "idle", adapterType: "process", createdAt: now, lastHeartbeatAt: now,
        runtimeConfig: { heartbeat: { enabled: true, intervalSec: 450, maxDailyRuns: 1 } } }).returning();
      await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", invocationSource: "timer",
        startedAt: now, finishedAt: now, costAccountingPending: false });
      const schedule = await createHeartbeatTimerPolicy(db).readTimerHeartbeatSchedule(agent, now);
      expect(schedule.deadline).toBe(Date.parse(deadline));
      expect(schedule.cap).toMatchObject({ reason: "heartbeat.daily_run_limit", observed: 1, limit: 1 });
    } finally {
      await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
      await db.delete(agents).where(eq(agents.id, agentId));
      await db.delete(companies).where(eq(companies.id, companyId));
    }
  });

  it("projects durable timers across client reconnection and executes the original plugin and trace owners", async () => {
    const now = Date.now(), companyId = randomUUID(), agentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID();
    let hold: { ownerId: string; expiresAt: number } = { ownerId: randomUUID(), expiresAt: now + 300_000 };
    let pluginReceipt: Awaited<ReturnType<NonNullable<PluginWorkerManager["inspectIdleSleep"]>>> | undefined;
    const capability = { deadlineCatalog: "domain_deadlines_v1" } as VerifiedIdleCapabilities;
    let projectionReads = 0;
    const read = (quietPluginIds: string[] = []) => db.transaction(async tx => {
      projectionReads = 0;
      await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
      const counted = { select: (...args: unknown[]) => { projectionReads++; return Reflect.apply(tx.select, tx, args); } } as Pick<Db, "select">;
      return readIdleDeadlineProof(counted, hold, { now: Date.now(), verifiedCapability: capability, quietPluginIds,
        quietPluginGeneration: pluginReceipt?.catalogGeneration, quietPluginCatalog: pluginReceipt?.catalog });
    }, { isolationLevel: "repeatable read", accessMode: "read only" });
    await db.insert(companies).values({ id: companyId, name: "Deadline fixture", issuePrefix: `D${companyId.slice(0, 6)}` });
    await db.insert(agents).values({ id: agentId, companyId, name: "Timer", role: "engineer", status: "idle", adapterType: "codex_local",
      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } }, createdAt: new Date(now), lastHeartbeatAt: new Date(now) });
    const central = (enabled: boolean, diagnostics = false) => readIdleSleepSafety(db, () => ({ ...hold, ownerId: hold.ownerId,
      draining: true, startedAt: new Date(now - 1000), expiresAt: new Date(hold.expiresAt), activeRuns: 0, pendingWakes: 0 }),
      () => now, hold.ownerId, async () => "none", undefined, undefined, enabled, enabled,
      enabled ? { deadlineCapability: capability, ...(diagnostics ? { collectDiagnostics: true as const } : {}) } : {});
    expect((await central(false)).backgroundWork).toBe("present");
    const negotiated = await central(true);
    expect(negotiated).toMatchObject({ version: 3, backgroundWork: "none", deadlineProof: { catalog: "domain_deadlines_v1" },
      scheduledWake: { version: 2, kind: "domain_deadlines" } });
    expect(negotiated.deadlineProof?.generation).toBe(negotiated.scheduledWake?.generation);
    const initial = await read();
    expect(initial.blocked).toBe(false);
    expect(initial.scheduledWake?.nextWakeAt).toBe(new Date(now + 600_000).toISOString());
    expect((await db.select().from(agents).where(eq(agents.id, agentId)))[0].lastHeartbeatAt?.getTime()).toBe(now);
    await db.$client.end({ timeout: 1 });
    db = createDb(external ?? temporary!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
    expect((await read()).scheduledWake).toEqual(initial.scheduledWake);
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 450 } } }).where(eq(agents.id, agentId));
    const earlier = await read();
    expect(earlier.generation).not.toBe(initial.generation);
    expect(earlier.scheduledWake?.nextWakeAt).toBe(new Date(now + 450_000).toISOString());
    await db.update(agents).set({ lastHeartbeatAt: new Date(now - 600_000) }).where(eq(agents.id, agentId));
    expect((await read()).blocked).toBe(true);
    const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
    const claimed = await Promise.all([claimDueTimerHeartbeatForAgent(db, agent, new Date(now), 450), claimDueTimerHeartbeatForAgent(db, agent, new Date(now), 450)]);
    expect(claimed.filter(Boolean)).toHaveLength(1);
    expect((await read()).scheduledWake?.nextWakeAt).toBe(new Date(now + 450_000).toISOString());
    await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", invocationSource: "timer",
      startedAt: new Date(now), finishedAt: new Date(now), costAccountingPending: false });
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 450, maxDailyRuns: 1 } } }).where(eq(agents.id, agentId));
    const date = new Date(now);
    const nextDay = Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate() + 1);
    expect((await read()).scheduledWake?.nextWakeAt).toBe(new Date(Math.max(now + 450_000, nextDay)).toISOString());
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 450, skipTimerWhenNoActionableWork: true } } }).where(eq(agents.id, agentId));
    expect((await read()).scheduledWake).toBeUndefined();
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: "true", intervalSec: 450 } } }).where(eq(agents.id, agentId));
    expect((await read()).blocked).toBe(true);
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: false } } }).where(eq(agents.id, agentId));
    expect((await read()).scheduledWake).toBeUndefined();
    const noAlarm = await central(true);
    expect(noAlarm).toMatchObject({ version: 3, backgroundWork: "none", deadlineProof: { catalog: "domain_deadlines_v1" } });
    expect(noAlarm.scheduledWake).toBeUndefined();
    expect(noAlarm.deadlineProof?.generation).not.toBe(negotiated.deadlineProof?.generation);
    const scaleIds = Array.from({ length: 160 }, () => randomUUID());
    const scaleAgent = (id: string, cost = false) => ({ id, companyId, name: "Scale timer", role: "engineer", status: "idle", adapterType: "codex_local",
      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 900, maxDailyRuns: 2, ...(cost ? { maxDailyCostCents: 100 } : {}) } },
      createdAt: new Date(now), lastHeartbeatAt: new Date(now) });
    await db.insert(agents).values(scaleIds.slice(0, 80).map(id => scaleAgent(id)));
    expect((await read()).scheduledWake?.nextWakeAt).toBe(new Date(now + 900_000).toISOString());
    expect(projectionReads).toBe(5);
    await db.insert(agents).values(scaleIds.slice(80).map(id => scaleAgent(id, true)));
    const inactiveIds = [randomUUID(), randomUUID(), randomUUID()];
    await db.insert(agents).values(inactiveIds.map((id, i) => ({ ...scaleAgent(id), status: i === 0 ? "paused" : i === 1 ? "terminated" : "idle",
      ...(i === 2 ? { runtimeConfig: { heartbeat: { enabled: false } } } : {}) })));
    const scaled = await read();
    expect(scaled.blocked).toBe(false);
    expect(scaled.scheduledWake?.nextWakeAt).toBe(new Date(now + 900_000).toISOString());
    expect(projectionReads).toBe(6);
    await db.update(agents).set({ lastHeartbeatAt: new Date(now - 900_000) }).where(eq(agents.id, scaleIds[0]));
    expect((await read()).blocked).toBe(true);
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: false } } }).where(inArray(agents.id, [...scaleIds, ...inactiveIds]));
    const cancelledScale = await read();
    expect(cancelledScale.scheduledWake).toBeUndefined();
    expect(cancelledScale.generation).not.toBe(scaled.generation);
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0.5 } }, lastHeartbeatAt: new Date(now) }).where(eq(agents.id, agentId));
    let checkpointClock = now;
    const becameDue = await readIdleSleepSafety(db, () => ({ ownerId: hold.ownerId, draining: true, startedAt: new Date(now - 1000),
      expiresAt: new Date(hold.expiresAt), activeRuns: 0, pendingWakes: 0 }), () => checkpointClock, hold.ownerId, async () => "none", undefined,
      async () => { const finish = beginIdleTrackedWork(); try { checkpointClock += 1000; return true; } finally { finish(); } },
      true, true, { deadlineCapability: capability });
    expect(becameDue.backgroundWork).toBe("present");
    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: false } } }).where(eq(agents.id, agentId));

    const manifest = { id: `deadline-${pluginId}`, apiVersion: 1 as const, version: "1.0.0", displayName: "Fixture", description: "Deadline fixture", author: "Test",
      categories: [], capabilities: ["jobs.schedule" as const, "plugin.state.write" as const], entrypoints: { worker: "worker.cjs" },
      jobs: [{ jobKey: "hourly", displayName: "Hourly", schedule: "0 * * * *" }] };
    await db.insert(plugins).values({ id: pluginId, pluginKey: manifest.id, packageName: "fixture-plugin", version: manifest.version, status: "ready", manifestJson: manifest });
    const store = pluginJobStore(db), state = pluginStateStore(db), manager = createPluginWorkerManager();
    let allowed = true;
    const errors: unknown[] = [];
    const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => allowed, onError: error => errors.push(error) });
    const scheduler = createPluginJobScheduler({ db, jobStore: store, workerManager: manager, isBackgroundWorkEnabled: () => allowed, tickIntervalMs: 10 });
    const prepare = async () => {
      allowed = false;
      const lease = startTaskDrain({ purpose: "idle", ttlMs: 300_000 });
      hold = { ownerId: lease.ownerId!, expiresAt: lease.expiresAt!.getTime() };
      pluginReceipt = await manager.inspectIdleSleep!({ ...hold });
      expect(pluginReceipt.backgroundWork).toBe("none");
      return read([pluginId]);
    };
    const resume = () => { stopTaskDrain(); manager.releaseIdleSleep!(); allowed = true; };
    try {
      await manager.startWorker(pluginId, { manifest, apiVersion: 1, config: {}, instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
        entrypointPath: fileURLToPath(new URL("./fixtures/plugin-worker-deadline.cjs", import.meta.url)),
        execArgv: ["--import", createRequire(import.meta.url).resolve("tsx")],
        hostHandlers: { "state.set": async input => {
          if (input.scopeKind !== "instance") throw new Error("Fixture job must publish instance-scoped state");
          await state.set(pluginId, { ...input, scopeKind: "instance" });
        } } });
      await store.syncJobDeclarations(pluginId, manifest.jobs);
      const [job] = await db.select().from(pluginJobs).where(eq(pluginJobs.pluginId, pluginId));
      expect((await prepare()).blocked).toBe(true);
      resume();
      await scheduler.start(coordinator);
      await scheduler.tick();
      await store.updateRunTimestamps(job.id, new Date(now - 3_600_000), new Date(now + 900_000));
      expect((await read()).blocked).toBe(true);
      const transferred = await prepare();
      expect(transferred.scheduledWake?.nextWakeAt).toBe(new Date(now + 900_000).toISOString());
      resume();
      vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(now + 900_001);
      await Promise.all([scheduler.tick(), scheduler.tick()]);
      await expect.poll(async () => (await db.select().from(pluginJobRuns).where(eq(pluginJobRuns.jobId, job.id))).map(run => run.status), { timeout: 5_000 }).toEqual(["succeeded"]);
      const effects = await db.select().from(pluginState).where(eq(pluginState.pluginId, pluginId));
      const runs = await db.select().from(pluginJobRuns).where(eq(pluginJobRuns.jobId, job.id));
      expect(effects).toHaveLength(1); expect(effects[0].stateKey).toBe(runs[0].id);
      expect(runs[0].finishedAt).not.toBeNull();
      const [after] = await db.select().from(pluginJobs).where(eq(pluginJobs.id, job.id));
      expect(after.nextRunAt!.getTime()).toBeGreaterThan(now + 900_001);
      vi.useRealTimers();
      expect((await prepare()).scheduledWake?.nextWakeAt).toBe(after.nextRunAt!.toISOString());
      resume(); vi.useRealTimers();
      await store.updateJobStatus(job.id, "paused");
      expect(errors).toEqual([]);
    } finally { resume(); vi.useRealTimers(); await scheduler.stop(); await coordinator.stop(); await manager.stopAll(); }
    hold = { ownerId: randomUUID(), expiresAt: Date.now() + 300_000 };
    expect((await read()).scheduledWake).toBeUndefined();

    const traces = providerTraceStore(db);
    const capture = await traces.prepare({ companyId, runId, provider: "opencode", requestedBy: "fixture" });
    expect((await read()).blocked).toBe(true);
    const bytes = JSON.stringify({ kind: "trace_status", status: "complete", debugChannel: "typescript_opencode_native", debugSequence: 1, acknowledgedDebugSequence: 0 }) + "\n";
    await fs.writeFile(capture.path, bytes);
    expect((await traces.finalize(runId, companyId))?.status).toBe("complete");
    expect((await read()).scheduledWake?.nextWakeAt).toBe(new Date(capture.metadata.expiresAt).toISOString());
    const savedCapture = (await traces.getByRun(runId, companyId))!;
    await db.update(providerTraceRecords).set({ status: "deleted", deletedAt: new Date(), traceRef: `s3-segments-v1:${companyId}/${runId}/${randomUUID()}.ndjson` }).where(eq(providerTraceRecords.runId, runId));
    expect((await read()).blocked).toBe(true);
    await db.update(providerTraceRecords).set({ status: "complete", deletedAt: null, traceRef: savedCapture.traceRef }).where(eq(providerTraceRecords.runId, runId));
    await traces.remove(runId, companyId);
    await expect(fs.stat(capture.path)).rejects.toMatchObject({ code: "ENOENT" });
    expect((await read()).blocked).toBe(false);

    const expiringRunId = randomUUID();
    await db.insert(heartbeatRuns).values({ id: expiringRunId, companyId, agentId, status: "succeeded", invocationSource: "timer", startedAt: new Date(now), finishedAt: new Date(), costAccountingPending: false });
    const expiring = await traces.prepare({ companyId, runId: expiringRunId, provider: "opencode", requestedBy: "fixture" });
    await fs.writeFile(expiring.path, bytes);
    expect((await traces.finalize(expiringRunId, companyId))?.status).toBe("complete");
    await fs.unlink(expiring.path); await fs.mkdir(expiring.path);
    const expiry = new Date(new Date(expiring.metadata.expiresAt).getTime() + 1);
    await expect(traces.cleanupExpired(expiry)).rejects.toThrow();
    expect((await read()).blocked).toBe(true);
    await fs.rmdir(expiring.path); await fs.writeFile(expiring.path, bytes);
    await traces.cleanupExpired(expiry);
    const cleaned = await traces.getByRun(expiringRunId, companyId);
    expect(cleaned).toMatchObject({ status: "expired", traceRef: "00000000-0000-0000-0000-000000000000.ndjson" });
    await expect(fs.stat(expiring.path)).rejects.toMatchObject({ code: "ENOENT" });
    expect((await read()).blocked).toBe(false);
    const watched = randomUUID();
    await db.insert(issues).values({ id: watched, companyId, title: "Watchdog grace fixture", status: "done" });
    await db.insert(issueWatchdogs).values({ companyId, issueId: watched, watchdogAgentId: agentId, status: "active" });
    expect((await central(true, true)).diagnostics?.checks).toContainEqual({ reason: "issue_watchdogs", state: "present" });
    const mergeIssue = randomUUID();
    await db.insert(issues).values({ id: mergeIssue, companyId, title: "Merge confirmation fixture", status: "in_review", assigneeAgentId: agentId });
    await db.insert(issueThreadInteractions).values({ companyId, issueId: mergeIssue, kind: "request_confirmation", status: "pending",
      title: "Merge fixture/repo#39?", payload: { version: 1, prompt: "Merge fixture/repo#39?" } });
    expect((await central(true, true)).diagnostics?.checks).toContainEqual({ reason: "issues.schedule", state: "present" });
  }, 30_000);

  it("uses real vault refresh leases, rotates credentials and settles GitHub binding debt after wake", async () => {
    const now = Date.now(), companyId = randomUUID(), owner = "deadline-owner";
    await db.insert(companies).values({ id: companyId, name: "GitHub deadline", issuePrefix: `G${companyId.slice(0, 6)}` });
    const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_http", status: "active" }).returning();
    const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true,
      transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", credentialSource: "paperclip_vault", transportConfig: {},
      config: { sourceTemplateKey: "github", oauth: { strategy: "paperclip_cloud_connector", connectorProfile: "github.code" } } }).returning();
    const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: "user", subjectUserId: owner, status: "active",
      credentialSecretRefs: [{ configPath: "oauth.access_token", secretId: randomUUID(), versionSelector: "latest" }],
      providerTenant: { oauth: { strategy: "paperclip_cloud_connector", accessTokenExpiresAt: new Date(now + 8 * 3_600_000).toISOString(), refreshedAt: new Date(now).toISOString() },
        github: { userId: "42", login: "fixture", installationCount: 1, repositoryCount: 1, repositorySelection: "all", installationIds: ["101"],
          installationOwnerLogins: ["fixture"], accessRevision: randomUUID(), lastAccessRefreshAt: new Date(now).toISOString(), webhookHealth: "healthy" } } }).returning();
    const read = () => db.transaction(async tx => {
      await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
      return readIdleGitHubGrantContinuity(tx, Date.now());
    }, { isolationLevel: "repeatable read", accessMode: "read only" });
    expect((await read()).blocked).toBe(true);
    const refs = [];
    for (const [configPath, value] of [["oauth.access_token", "saved-access"], ["oauth.refresh_token", "saved-refresh"]] as const) {
      const saved = await writeConnectionCredential(db, { companyId, connectionName: "GitHub", configPath, label: configPath, value, ownerUserId: owner });
      refs.push({ configPath, secretId: saved.secret.id, versionSelector: "latest" as const, required: true });
    }
    await db.update(connectionGrants).set({ credentialSecretRefs: refs }).where(eq(connectionGrants.id, grant.id));
    const vault = secretService(db), credentials = createOAuthCredentialStore(db, vault);
    await credentials.syncCredentialBindings(connection, refs);
    const pinned = await writeConnectionCredential(db, { companyId, connectionName: "GitHub pinned", configPath: "oauth.access_token",
      label: "Pinned access", value: "pinned-initial", ownerUserId: owner });
    const pinnedRef = { configPath: "oauth.access_token", secretId: pinned.secret.id, versionSelector: 1 as const, required: true };
    await writeConnectionCredential(db, { companyId, connectionName: "GitHub pinned", configPath: "oauth.access_token", label: "Pinned access",
      value: "pinned-rotated", ownerUserId: owner, existingRef: pinnedRef });
    await db.update(connectionGrants).set({ credentialSecretRefs: [pinnedRef, refs[1]] }).where(eq(connectionGrants.id, grant.id));
    await credentials.syncCredentialBindings(connection, [pinnedRef, refs[1]]);
    const [pinnedGrant] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant.id));
    expect((await credentials.resolveOAuthGrantSecret(connection, pinnedGrant, pinnedRef,
      { actorType: "system", actorId: "fixture" }, undefined)).value).toBe("pinned-initial");
    expect((await read()).blocked).toBe(false);
    for (const status of ["disabled", "destroyed"]) {
      await db.update(companySecretVersions).set({ status }).where(eq(companySecretVersions.secretId, pinned.secret.id));
      expect((await read()).blocked).toBe(true);
    }
    await db.update(companySecretVersions).set({ status: "previous", revokedAt: new Date() }).where(eq(companySecretVersions.secretId, pinned.secret.id));
    expect((await read()).blocked).toBe(true);
    await db.update(companySecretVersions).set({ revokedAt: null }).where(eq(companySecretVersions.secretId, pinned.secret.id));
    expect((await read()).blocked).toBe(false);
    await db.update(connectionGrants).set({ credentialSecretRefs: refs }).where(eq(connectionGrants.id, grant.id));
    await credentials.syncCredentialBindings(connection, refs);
    const baseline = await read();
    expect(baseline.blocked).toBe(false);
    expect(baseline.connectionIds).toContain(connection.id);
    expect(new Date(baseline.deadlines[0]).toISOString()).toBe(new Date(now + 7 * 3_600_000).toISOString());
    const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, connection.id)))[0];
    const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, credentials.getOAuthGrantRow);
    let failBindings = true, rotations = 0;
    const successfulBindings = new Map<string, { active: boolean; grantId: string }>();
    const connector = {
      refresh: async (input: Parameters<PaperclipCloudConnector["refresh"]>[0]) => {
        rotations++; expect(input.refreshToken).toBe("saved-refresh"); expect(input.subject).toBe(owner);
        return { v: 1 as const, accessToken: "renewed-access", refreshToken: "renewed-refresh", tokenType: "Bearer", accessTokenExpiresAt: new Date(Date.now() + 8 * 3_600_000).toISOString(),
          refreshTokenExpiresAt: null, scopes: [...GITHUB_CONNECTOR_PROFILES["github.code"].scopes], subject: owner, companyId, instanceId: "fixture", environment: "test" as const, provider: "github" as const, profile: "github.code" };
      },
      setWebhookBinding: async (input: Parameters<PaperclipCloudConnector["setWebhookBinding"]>[0]) => {
        expect(input.accessToken).toBe("renewed-access"); expect(input.companyId).toBe(companyId);
        if (failBindings) throw new Error("named Cloud binding provider unavailable");
        successfulBindings.set(input.id, { active: input.active, grantId: input.grantId });
      },
    } as PaperclipCloudConnector;
    const refresh = createOAuthGrantRefresh(db, { now: () => new Date(), store: credentials, leases, getConnectionRow: getConnection,
      currentCloudConnector: () => connector, exchangeOAuthToken: async () => { throw new Error("Direct OAuth must not run for managed GitHub"); } });
    const continuity = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector,
      refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret });
    vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
      const url = new URL(String(input)); expect(url.origin).toBe("https://api.github.com");
      expect(new Headers(init?.headers).get("authorization")).toBe("Bearer renewed-access");
      const body = url.pathname === "/user" ? { id: 42, login: "fixture" }
        : url.pathname === "/user/installations" ? { installations: [{ id: 101, repository_selection: "all", account: { login: "fixture" } }] }
        : { repositories: [{ id: 999, full_name: "fixture/repo" }] };
      return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } });
    });
    try {
      vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(now + 7 * 3_600_000 + 1);
      expect(await continuity.sweepGitHubConnectionContinuity(companyId)).toMatchObject({ due: 1, failed: 1 });
      expect(rotations).toBe(1);
      const [debt] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant.id));
      expect(debt.providerTenant?.github?.bindingSync).toBeDefined();
      expect(debt.providerTenant?.github?.continuityRetry).toBeDefined();
      expect((await read()).blocked).toBe(true);
      failBindings = false; vi.setSystemTime(Date.now() + 31_000);
      const reconstructed = createGitHubGrantContinuity(db, { now: () => new Date(), currentCloudConnector: () => connector,
        refreshOAuthGrantCredentials: refresh, resolveOAuthGrantSecret: credentials.resolveOAuthGrantSecret });
      expect(await reconstructed.sweepGitHubConnectionContinuity(companyId)).toMatchObject({ due: 1, failed: 0 });
      expect(rotations).toBe(1); expect(successfulBindings.size).toBe(1);
      const [settled] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant.id));
      expect(settled.providerTenant?.github?.bindingSync).toBeUndefined();
      expect(settled.providerTenant?.github?.continuityRetry).toBeUndefined();
      const [stored] = await db.select().from(companySecrets).where(eq(companySecrets.id, refs[0].secretId));
      expect(stored.latestVersion).toBe(2);
      expect((await credentials.resolveOAuthGrantSecret(connection, settled, refs[0], { actorType: "system", actorId: "fixture" }, undefined)).value).toBe("renewed-access");
      const after = await read();
      expect(after.blocked).toBe(false); expect(after.connectionIds).toContain(connection.id);
      expect(Math.min(...after.deadlines)).toBeGreaterThan(Date.now());
    } finally { vi.useRealTimers(); vi.restoreAllMocks(); }
  }, 30_000);

});