Expose signed routine wake deadlines during idle sleep checks

Expose signed routine wake deadlines during idle sleep checks

Report the earliest eligible routine deadline to an authenticated host scheduler, while preserving local execution, owned sleep leases and conservative blocking for unsupported work.

Verified with focused tests, typecheck, build, all required CI checks and Apex 5/5.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/idle-scheduled-work.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it, vi } from "vitest";
+import type { Db } from "@paperclipai/db";
+import { readIdleRoutineSchedule } from "../services/idle-scheduled-work.js";
+
+const now = Date.UTC(2020, 0, 1);
+const trigger = { id: "fixture", routine_id: "routine", company_id: "company", kind: "schedule",
+  setup_pending: false, cron_expression: "*/15 * * * *", timezone: "UTC", scoped: true,
+  next_run_at: new Date(now + 600_000), routine_updated: new Date(now), trigger_updated: new Date(now) };
+function database(rows: unknown[]) {
+  return { execute: vi.fn().mockResolvedValueOnce([{ blocked: false }]).mockResolvedValueOnce(rows) } as unknown as Pick<Db, "execute">;
+}
+describe("bounded idle routine catalog", () => {
+  it("validates shared timezones once in a full-size catalog", async () => {
+    const formatter = vi.spyOn(Intl, "DateTimeFormat");
+    try {
+      const rows = Array.from({ length: 10_000 }, (_, index) => ({ ...trigger, id: String(index) }));
+      const result = await readIdleRoutineSchedule(database(rows), "owner", now);
+      expect(result.blocked).toBe(false);
+      expect(result.scheduledWake?.nextWakeAt).toBe(trigger.next_run_at.toISOString());
+      expect(formatter).toHaveBeenCalledOnce();
+    } finally { formatter.mockRestore(); }
+  });
+  it("refuses an oversized catalog before expensive timezone validation", async () => {
+    const formatter = vi.spyOn(Intl, "DateTimeFormat");
+    try {
+      expect(await readIdleRoutineSchedule(database(Array(10_001).fill(trigger)), "owner", now))
+        .toEqual({ blocked: true });
+      expect(formatter).not.toHaveBeenCalled();
+    } finally { formatter.mockRestore(); }
+  });
+});

server/src/__tests__/idle-sleep-safety.test.ts

@@ -6,7 +6,7 @@ import {
   agentInstructionWorkingCopies, agents, agentApiKeys, agentWakeupRequests, companies, companySecretProposals, createDb,
   adapterAuthSessions, environments, environmentLeases, executionWorkspaces,
   heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries,
-  projects, projectWorkspaces, routines, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,
+  projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,
 } from "@paperclipai/db";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
@@ -175,6 +175,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
   afterAll(async () => { await database?.cleanup(); });
 
   const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal);
+  const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);
   async function seed() {
     const companyId = randomUUID();
     const agentId = randomUUID();
@@ -185,6 +186,84 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
     return { companyId, agentId, runId };
   }
 
+  async function scheduledRoutine(at = now + 600_000) {
+    const fixture = await seed();
+    const [routine] = await db.insert(routines).values({ companyId: fixture.companyId,
+      title: "Scheduled fixture", status: "active", assigneeAgentId: fixture.agentId }).returning();
+    const [trigger] = await db.insert(routineTriggers).values({ companyId: fixture.companyId,
+      routineId: routine!.id, kind: "schedule", cronExpression: "*/15 * * * *", timezone: "UTC",
+      nextRunAt: new Date(at), enabled: true, archived: false, setupPending: false }).returning();
+    return { ...fixture, routine: routine!, trigger: trigger! };
+  }
+
+  it("scheduled routines require the opt-in report and retain their durable definitions", async () => {
+    const f = await scheduledRoutine();
+    expect(await read()).toEqual(present);
+    const result = await readScheduled();
+    expect(result).toMatchObject({ version: 2, backgroundWork: "none", scheduledWake: {
+      kind: "routines", version: 1, nextWakeAt: new Date(now + 600_000).toISOString(),
+    } });
+    expect(result.scheduledWake?.generation).toMatch(/^[a-f0-9]{64}$/);
+    expect(await db.select().from(routineTriggers).where(eq(routineTriggers.id, f.trigger.id)))
+      .toMatchObject([{ nextRunAt: f.trigger.nextRunAt, cronExpression: f.trigger.cronExpression }]);
+    expect((await readScheduled()).scheduledWake).toEqual(result.scheduledWake);
+  });
+  it("scheduled routines cover all companies and invalidate a moved-earlier alarm", async () => {
+    await scheduledRoutine(now + 900_000);
+    const f = await scheduledRoutine(now + 600_000);
+    const first = await readScheduled();
+    expect(first.scheduledWake?.nextWakeAt).toBe(new Date(now + 600_000).toISOString());
+    await db.update(routineTriggers).set({ nextRunAt: new Date(now + 300_000) }).where(eq(routineTriggers.id, f.trigger.id));
+    const edited = await readScheduled();
+    expect(edited.scheduledWake?.nextWakeAt).toBe(new Date(now + 300_000).toISOString());
+    expect(edited.scheduledWake?.generation).not.toBe(first.scheduledWake?.generation);
+  });
+  it.each([
+    { kind: "webhook" }, { setupPending: true }, { nextRunAt: null },
+    { nextRunAt: new Date(now) }, { cronExpression: "bad" }, { timezone: "Unknown/Zone" },
+  ])("scheduled routines retain unsupported or due work: %j", async change => {
+    const f = await scheduledRoutine();
+    await db.update(routineTriggers).set(change).where(eq(routineTriggers.id, f.trigger.id));
+    expect(await readScheduled()).toEqual(present);
+  });
+  it("scheduled routines cannot exempt external API credentials or an unowned hold", async () => {
+    const f = await scheduledRoutine();
+    expect(await readIdleSleepSafety(db, owned, () => now, "stale", emptyLocal, undefined, undefined, true)).toEqual(unknown);
+    await db.insert(agentApiKeys).values({ companyId: f.companyId, agentId: f.agentId,
+      name: "Fixture external automation", keyHash: "fixture-hash" });
+    expect(await readScheduled()).toEqual(present);
+  });
+  it("scheduled routines ignore disabled triggers and fail closed on unknown routine states", async () => {
+    const f = await scheduledRoutine();
+    await db.update(routineTriggers).set({ enabled: false }).where(eq(routineTriggers.id, f.trigger.id));
+    expect(await readScheduled()).toEqual(none);
+    await db.update(routines).set({ status: "unknown" }).where(eq(routines.id, f.routine.id));
+    expect(await readScheduled()).toEqual(present);
+  });
+  it("scheduled routines reject a trigger with a different company", async () => {
+    const f = await scheduledRoutine();
+    const other = await seed();
+    await db.update(routineTriggers).set({ companyId: other.companyId }).where(eq(routineTriggers.id, f.trigger.id));
+    expect(await readScheduled()).toEqual(present);
+  });
+  it.each(["checkpoint", "local inspection"])("keeps a routine awake when it becomes due during %s", async stage => {
+    await scheduledRoutine(now + 1000);
+    let clock = now;
+    let inspections = 0;
+    const inspect = async () => {
+      if (++inspections === 2 && stage === "local inspection") clock += 1000;
+      return "none" as const;
+    };
+    const checkpoint = stage === "checkpoint" ? async () => {
+      const done = beginIdleTrackedWork();
+      clock += 1000;
+      done();
+      return true;
+    } : undefined;
+    expect(await readIdleSleepSafety(db, owned, () => clock, ownerId, inspect, undefined, checkpoint, true)).toEqual(present);
+    expect(idleWorkSnapshot().active).toBe(0);
+  });
+
   it.each(["succeeded", "failed", "cancelled", "timed_out", "interrupted"])(
     "permits a finished %s run with a consumed retry date without changing history", async status => {
       const { runId } = await seed();

server/src/__tests__/instance-settings-routes.test.ts

@@ -907,7 +907,7 @@ describe("instance settings routes", () => {
       await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);
       expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });
       await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
-      expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined);
+      expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false);
     });
 
     it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {
@@ -934,10 +934,23 @@ describe("instance settings routes", () => {
       const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");
       expect(res.status).toBe(200);
       expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });
-      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined);
+      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false);
       expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);
     });
 
+    it("reserves routine deadline reports for verified Cloud control callers", async () => {
+      mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
+      for (const source of ["session", "cloud_tenant", "local_implicit"]) {
+        await request(createApp({ ...adminActor, source }))
+          .get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(403);
+      }
+      expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
+      await request(createApp({ ...adminActor, source: "cloud_control" }))
+        .get("/api/instance/task-drain?idleSleepSafety=1&scheduledWake=1&ownerId=fixture-owner").expect(200);
+      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
+        "fixture-owner", expect.any(Function), undefined, undefined, true);
+    });
+
     it("uses the live worker manager for inspection and releases only the owned hold", async () => {
       const inspect = vi.fn(async () => ({ backgroundWork: "none", pluginIds: [] }));
       const release = vi.fn();
@@ -970,7 +983,7 @@ describe("instance settings routes", () => {
       expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
       await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
       expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
-        "fixture-owner", expect.any(Function), undefined, backup);
+        "fixture-owner", expect.any(Function), undefined, backup, false);
       mockReadIdleSleepSafety.mockClear();
       await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);
       expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
@@ -992,7 +1005,7 @@ describe("instance settings routes", () => {
         const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup);
         await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
         expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
-          "fixture-owner", expect.any(Function), undefined, backup);
+          "fixture-owner", expect.any(Function), undefined, backup, false);
       } finally {
         delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
       }

server/src/routes/instance-settings.ts

@@ -305,6 +305,10 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag
       // The report covers every company in this process. Ordinary company
       // members may read process counters, but not instance-wide work state.
       assertCanManageInstanceSettings(req);
+      const scheduledRoutines = req.query.scheduledWake === "1";
+      if (scheduledRoutines && req.actor.source !== "cloud_control") {
+        throw forbidden("Scheduled wake reports require Cloud control authority");
+      }
       // A checkpoint performs platform-owned backup work. Tenant owners can
       // have instance-admin elevation, but that must not grant the manual
       // backup authority which managed instances deliberately withhold.
@@ -316,7 +320,7 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag
       const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now,
         typeof req.query.ownerId === "string" ? req.query.ownerId : undefined,
         () => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }),
-        pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup);
+        pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines);
       res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety });
       return;
     }

server/src/routes/openapi.ts

@@ -6852,7 +6852,7 @@ registry.registerPath({
   tags: ["instance"],
   summary:
     "Get the task-drain status for this process only; quiescent counts in-process work, and a process restart clears it even when the database still holds running rows",
-  request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep.") }) },
+  request: { query: z.object({ ownerId: z.string().uuid().optional(), idleSleepSafety: z.literal("1").optional().describe("Instance admins can request a conservative durable-work report while admission and ingress are held. Unknown or present work must prevent automatic sleep."), scheduledWake: z.literal("1").optional().describe("Cloud control callers may request an owned routine deadline report. Cloud must acknowledge a durable alarm and revalidate this report before stopping the instance.") }) },
   responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden },
 });
 

server/src/services/idle-scheduled-work.ts

@@ -0,0 +1,62 @@
+import { createHash, randomUUID } from "node:crypto";
+import { sql } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import { validateCron } from "./cron.js";
+
+const processGeneration = randomUUID();
+const limit = 10_000;
+
+export interface ScheduledWake {
+  version: 1;
+  kind: "routines";
+  generation: string;
+  nextWakeAt: string;
+}
+
+export async function readIdleRoutineSchedule(tx: Pick<Db, "execute">, ownerId: string, now: number): Promise<{
+  blocked: boolean;
+  scheduledWake?: ScheduledWake;
+}> {
+  const unknown = await tx.execute<{ blocked: boolean }>(sql`SELECT EXISTS (
+    SELECT 1 FROM routines WHERE status NOT IN ('active', 'paused', 'archived')
+  ) AS blocked`);
+  if (unknown.length !== 1 || unknown[0]?.blocked !== false) return { blocked: true };
+  const rows = await tx.execute<{
+    id: string; routine_id: string; company_id: string; kind: string;
+    setup_pending: boolean; cron_expression: string | null; timezone: string | null;
+    next_run_at: Date | string | null; routine_updated: Date | string; trigger_updated: Date | string;
+    scoped: boolean;
+  }>(sql`SELECT t.id, t.routine_id, t.company_id, t.kind, t.setup_pending,
+      t.cron_expression, t.timezone, t.next_run_at,
+      r.updated_at AS routine_updated, t.updated_at AS trigger_updated,
+      (t.company_id = r.company_id) AS scoped
+    FROM routine_triggers t JOIN routines r ON r.id = t.routine_id
+    WHERE r.status = 'active' AND t.enabled AND NOT t.archived
+    ORDER BY t.id LIMIT ${limit + 1}`);
+  if (rows.length > limit) return { blocked: true };
+  let earliest = Infinity;
+  const validTimezones = new Set<string>();
+  const validCrons = new Set<string>();
+  for (const row of rows) {
+    if (row.scoped !== true || row.kind !== "schedule" || row.setup_pending !== false
+      || !row.cron_expression || !row.timezone
+      || row.next_run_at === null) return { blocked: true };
+    if (!validCrons.has(row.cron_expression)) {
+      if (validateCron(row.cron_expression)) return { blocked: true };
+      validCrons.add(row.cron_expression);
+    }
+    if (!validTimezones.has(row.timezone)) {
+      try { new Intl.DateTimeFormat("en", { timeZone: row.timezone }); }
+      catch { return { blocked: true }; }
+      validTimezones.add(row.timezone);
+    }
+    const at = new Date(row.next_run_at).getTime();
+    if (!Number.isFinite(at) || at <= now) return { blocked: true };
+    earliest = Math.min(earliest, at);
+  }
+  if (!Number.isFinite(earliest)) return { blocked: false };
+  const generation = createHash("sha256").update(JSON.stringify([processGeneration, ownerId, rows])).digest("hex");
+  return { blocked: false, scheduledWake: {
+    version: 1, kind: "routines", generation, nextWakeAt: new Date(earliest).toISOString(),
+  } };
+}

server/src/services/idle-sleep-safety.ts

@@ -1,5 +1,6 @@
 import { idleWorkSnapshot } from "./task-admission.js";
 import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js";
+import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js";
 import type { Db } from "@paperclipai/db";
 import { sql } from "drizzle-orm";
 
@@ -13,8 +14,9 @@ export interface IdleSleepDrainStatus {
 }
 
 export interface IdleSleepSafety {
-  version: 1;
+  version: 1 | 2;
   backgroundWork: "none" | "present" | "unknown";
+  scheduledWake?: ScheduledWake;
 }
 
 export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{
@@ -219,6 +221,7 @@ export async function readIdleSleepSafety(
   inspectLocalWork: () => Promise<IdleLocalWork> = readIdleLocalWork,
   inspectPlugins?: InspectIdlePlugins,
   prepareBackup?: () => Promise<boolean>,
+  scheduledRoutines = false,
 ): Promise<IdleSleepSafety> {
   const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" };
   const before = getDrainStatus();
@@ -236,16 +239,27 @@ export async function readIdleSleepSafety(
       if (pluginWork.backgroundWork !== "none") return { version: 1, backgroundWork: pluginWork.backgroundWork };
       quietPlugins = pluginWork.pluginIds;
     }
+    let scheduledWake: ScheduledWake | undefined;
     const blocked = await db.transaction(async (tx) => {
       await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
-      const checks = WORK_CHECKS.map((query) => sql`EXISTS (${sql.raw(query)})`);
+      const checks = WORK_CHECKS.filter(query => !scheduledRoutines ||
+        query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`)
+        .map((query) => sql`EXISTS (${sql.raw(query)})`);
       // Only the current worker's owned runtime drain can exempt a plugin.
       // Missing workers, old SDKs, and newly installed plugins still block.
       checks.push(quietPlugins.length === 0
         ? sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled')`
         : sql`EXISTS (SELECT 1 FROM plugins WHERE status <> 'disabled' AND id NOT IN (${sql.join(quietPlugins.map((id) => sql`${id}::uuid`), sql`, `)}))`);
       const rows = await tx.execute<{ blocked: boolean }>(sql`SELECT ${sql.join(checks, sql` OR `)} AS blocked`);
-      return rows.length === 1 && typeof rows[0]?.blocked === "boolean" ? rows[0].blocked : undefined;
+      if (rows.length !== 1 || typeof rows[0]?.blocked !== "boolean") return undefined;
+      if (rows[0].blocked) return true;
+      if (scheduledRoutines) {
+        if (!ownerId || ownerId !== before.ownerId) return undefined;
+        const schedule = await readIdleRoutineSchedule(tx, ownerId, now());
+        if (schedule.blocked) return true;
+        scheduledWake = schedule.scheduledWake;
+      }
+      return false;
     }, { isolationLevel: "repeatable read", accessMode: "read only" });
     if (blocked === undefined || !sameQuietHold(before, getDrainStatus(), now())) return unknown;
     if (blocked) return { version: 1, backgroundWork: "present" };
@@ -265,6 +279,10 @@ export async function readIdleSleepSafety(
     const localAfter = idleWorkSnapshot();
     if (!sameQuietHold(before, after, now()) || localAfter.active !== 0 ||
         expectedGeneration !== localAfter.generation) return unknown;
+    if (local === "none" && scheduledWake) {
+      if (Date.parse(scheduledWake.nextWakeAt) <= now()) return { version: 1, backgroundWork: "present" };
+      return { version: 2, backgroundWork: "none", scheduledWake };
+    }
     return { version: 1, backgroundWork: local };
 
   } catch {