Merge health warning repair into command scheduling

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/idle-tool-connection-health.md

@@ -116,3 +116,10 @@ rows uses `tool_connections_company_enabled_idx`, 13 shared buffer hits, and
 0.022 ms execution; the diagnostic data was rolled back. No new index is needed
 for this observed query. The local passing selection totals 770 distinct tests.
 Full workspace and remote acceptance gates remain required.
+
+Review repair: a controlled 503 probe persisted its failed health observation,
+but the warning regression failed before the fix. Scheduled probe failures now
+log company/connection identifiers and a fixed message, without provider prose.
+The saved 15-minute deadline still controls the next check. All 64 health/startup
+cases and server typechecking pass after repair; total selected coverage is now
+771 unique tests. Fresh remote gates are required for this repair.

server/src/__tests__/tool-connection-health-work.test.ts

@@ -1,3 +1,4 @@
+import { logger } from "../middleware/logger.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
 import { deleteCompany } from "../services/company-deletion.js";
 import { notifyChatConnectionHealth } from "../services/chat-channels/connection-health-notifications.js";
@@ -257,4 +258,26 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     await vi.advanceTimersByTimeAsync(1_800_000); expect(probe).toHaveBeenCalledOnce();
   });
 
+  it("warns for a persisted failed probe without retrying before its saved deadline", async () => {
+    const warnings = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
+    const requests = vi.fn(async () => new Response("private-provider-response", { status: 503 }));
+    const tools = toolAccessService(db, {
+      remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 }], remoteHttpRequest: requests,
+    });
+    const row = await connection({ config: { url: "https://health.example.test/mcp" } });
+    await start(tools.scheduledHealth);
+    const [stored] = await db.select().from(toolConnections).where(eq(toolConnections.id, row.id));
+    expect(stored.healthCheckedAt).not.toBeNull();
+    expect(warnings).toHaveBeenCalledWith(
+      { companyId, connectionId: row.id }, "Scheduled tool connection health check failed",
+    );
+    expect(JSON.stringify(warnings.mock.calls)).not.toContain("private-provider-response");
+    const due = stored.healthCheckedAt!.getTime() + 900_000;
+    await idleAt(due);
+    const before = requests.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(180_000);
+    expect(requests).toHaveBeenCalledTimes(before);
+    expect(errors).toEqual([]);
+  });
+
 });

server/src/index.ts

@@ -1892,7 +1892,6 @@ async function startServerWithDatabaseTeardown(
           logger.error({ err }, "status-card scheduler tick failed");
         }));
 
-        if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
         if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
         if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
           // Periodically reap orphaned runs (5-min staleness threshold) and make sure

server/src/services/tool-access/catalog/health-schedule.ts

@@ -1,3 +1,4 @@
+import { logger } from "../../../middleware/logger.js";
 import { and, eq, ne, sql } from "drizzle-orm";
 import { toolApplications, toolConnections, type Db } from "@paperclipai/db";
 import { isConnectorFeatureEnabled } from "@paperclipai/shared";
@@ -39,7 +40,10 @@ export function connectionHealthSchedule(db: Db, checkHealth: ConnectionHealth["
     if (!before || before.dueAt > Date.now()) return before ?? null;
     let failure: unknown;
     try { await checkHealth(id, { actorType: "system", actorId: "tool_health_sweep" }); }
-    catch (error) { failure = error; }
+    catch (error) {
+      failure = error;
+      logger.warn({ companyId: before.companyId, connectionId: id }, "Scheduled tool connection health check failed");
+    }
     const [after] = await load(before.companyId, id);
     if (failure && after && after.dueAt <= Date.now()) throw failure;
     return after ?? null;