From da4089960619d3da2112a4eed3cc7a3d97dbebf8 Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Sun, 25 Oct 2026 00:40:05 -0500
Subject: [PATCH] fix(server): retain warnings for scheduled health failures

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 doc/idle-tool-connection-health.md            |  7 ++++++
 .../tool-connection-health-work.test.ts       | 23 +++++++++++++++++++
 server/src/index.ts                           |  1 -
 .../tool-access/catalog/health-schedule.ts    |  6 ++++-
 4 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/doc/idle-tool-connection-health.md b/doc/idle-tool-connection-health.md
index 1fbaad7bb42..4b2a067538d 100644
--- a/doc/idle-tool-connection-health.md
+++ b/doc/idle-tool-connection-health.md
@@ -116,3 +116,10 @@ rows uses `tool_connections_company_enabled_idx`, 13 shared buffer hits, and
 0.022 ms execution; the diagnostic data was rolled back. No new index is needed
 for this observed query. The local passing selection totals 770 distinct tests.
 Full workspace and remote acceptance gates remain required.
+
+Review repair: a controlled 503 probe persisted its failed health observation,
+but the warning regression failed before the fix. Scheduled probe failures now
+log company/connection identifiers and a fixed message, without provider prose.
+The saved 15-minute deadline still controls the next check. All 64 health/startup
+cases and server typechecking pass after repair; total selected coverage is now
+771 unique tests. Fresh remote gates are required for this repair.
diff --git a/server/src/__tests__/tool-connection-health-work.test.ts b/server/src/__tests__/tool-connection-health-work.test.ts
index da1bded5041..fa0b9d8dbc5 100644
--- a/server/src/__tests__/tool-connection-health-work.test.ts
+++ b/server/src/__tests__/tool-connection-health-work.test.ts
@@ -1,3 +1,4 @@
+import { logger } from "../middleware/logger.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
 import { deleteCompany } from "../services/company-deletion.js";
 import { notifyChatConnectionHealth } from "../services/chat-channels/connection-health-notifications.js";
@@ -257,4 +258,26 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     await vi.advanceTimersByTimeAsync(1_800_000); expect(probe).toHaveBeenCalledOnce();
   });
 
+  it("warns for a persisted failed probe without retrying before its saved deadline", async () => {
+    const warnings = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
+    const requests = vi.fn(async () => new Response("private-provider-response", { status: 503 }));
+    const tools = toolAccessService(db, {
+      remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 }], remoteHttpRequest: requests,
+    });
+    const row = await connection({ config: { url: "https://health.example.test/mcp" } });
+    await start(tools.scheduledHealth);
+    const [stored] = await db.select().from(toolConnections).where(eq(toolConnections.id, row.id));
+    expect(stored.healthCheckedAt).not.toBeNull();
+    expect(warnings).toHaveBeenCalledWith(
+      { companyId, connectionId: row.id }, "Scheduled tool connection health check failed",
+    );
+    expect(JSON.stringify(warnings.mock.calls)).not.toContain("private-provider-response");
+    const due = stored.healthCheckedAt!.getTime() + 900_000;
+    await idleAt(due);
+    const before = requests.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(180_000);
+    expect(requests).toHaveBeenCalledTimes(before);
+    expect(errors).toEqual([]);
+  });
+
 });
diff --git a/server/src/index.ts b/server/src/index.ts
index 88f7a00b85d..13c6362003b 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -1892,7 +1892,6 @@ async function startServerWithDatabaseTeardown(
           logger.error({ err }, "status-card scheduler tick failed");
         }));
 
-        if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
         if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
         if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
           // Periodically reap orphaned runs (5-min staleness threshold) and make sure
diff --git a/server/src/services/tool-access/catalog/health-schedule.ts b/server/src/services/tool-access/catalog/health-schedule.ts
index af6ca8c92b3..06a2781121d 100644
--- a/server/src/services/tool-access/catalog/health-schedule.ts
+++ b/server/src/services/tool-access/catalog/health-schedule.ts
@@ -1,3 +1,4 @@
+import { logger } from "../../../middleware/logger.js";
 import { and, eq, ne, sql } from "drizzle-orm";
 import { toolApplications, toolConnections, type Db } from "@paperclipai/db";
 import { isConnectorFeatureEnabled } from "@paperclipai/shared";
@@ -39,7 +40,10 @@ export function connectionHealthSchedule(db: Db, checkHealth: ConnectionHealth["
     if (!before || before.dueAt > Date.now()) return before ?? null;
     let failure: unknown;
     try { await checkHealth(id, { actorType: "system", actorId: "tool_health_sweep" }); }
-    catch (error) { failure = error; }
+    catch (error) {
+      failure = error;
+      logger.warn({ companyId: before.companyId, connectionId: id }, "Scheduled tool connection health check failed");
+    }
     const [after] = await load(before.companyId, id);
     if (failure && after && after.dueAt <= Date.now()) throw failure;
     return after ?? null;