diff --git a/server/src/__tests__/feedback-flush-controller.test.ts b/server/src/__tests__/feedback-flush-controller.test.ts new file mode 100644 index 00000000000..5fa0f2eb720 --- /dev/null +++ b/server/src/__tests__/feedback-flush-controller.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { isDatabaseConnectionUnavailableError } from "../app.js"; + +describe("feedback export flush error classification", () => { + it("recognizes wrapped database connection-refused errors", () => { + const error = new Error("Failed query: select ...: connect ECONNREFUSED 127.0.0.1:54329"); + (error as { cause?: unknown }).cause = Object.assign( + new Error("connect ECONNREFUSED 127.0.0.1:54329"), + { code: "ECONNREFUSED" }, + ); + + expect(isDatabaseConnectionUnavailableError(error)).toBe(true); + }); + + it("does not classify ordinary feedback upload failures as database outages", () => { + expect(isDatabaseConnectionUnavailableError(new Error("upstream returned 500"))).toBe(false); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index 21ebb1497d3..59390116d2f 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -82,6 +82,14 @@ const VITE_DEV_STATIC_PATHS = new Set([ "/sw.js", ]); +export function isDatabaseConnectionUnavailableError(err: unknown): boolean { + const error = err as { code?: unknown; message?: unknown; cause?: unknown }; + if (error?.code === "ECONNREFUSED") return true; + const message = typeof error?.message === "string" ? error.message : ""; + if (message.includes("ECONNREFUSED")) return true; + return Boolean(error?.cause && isDatabaseConnectionUnavailableError(error.cause)); +} + export function resolveViteHmrPort(serverPort: number): number { if (serverPort <= 55_535) { return serverPort + 10_000; @@ -405,18 +413,37 @@ export async function createApp( jobCoordinator.start(); scheduler.start(); - const feedbackExportTimer = opts.feedbackExportService + let feedbackExportShuttingDown = false; + let feedbackExportTimer: ReturnType | null = null; + const disableFeedbackExportFlushes = () => { + feedbackExportShuttingDown = true; + if (feedbackExportTimer) { + clearInterval(feedbackExportTimer); + feedbackExportTimer = null; + } + }; + const flushPendingFeedbackExports = async () => { + if (feedbackExportShuttingDown) return; + try { + await opts.feedbackExportService?.flushPendingFeedbackTraces(); + } catch (err) { + if (isDatabaseConnectionUnavailableError(err)) { + disableFeedbackExportFlushes(); + logger.warn({ err }, "Disabling pending feedback export flushes because the database is unavailable"); + return; + } + logger.error({ err }, "Failed to flush pending feedback exports"); + } + }; + + feedbackExportTimer = opts.feedbackExportService ? setInterval(() => { - void opts.feedbackExportService?.flushPendingFeedbackTraces().catch((err) => { - logger.error({ err }, "Failed to flush pending feedback exports"); - }); + void flushPendingFeedbackExports(); }, FEEDBACK_EXPORT_FLUSH_INTERVAL_MS) : null; feedbackExportTimer?.unref?.(); if (opts.feedbackExportService) { - void opts.feedbackExportService.flushPendingFeedbackTraces().catch((err) => { - logger.error({ err }, "Failed to flush pending feedback exports"); - }); + void flushPendingFeedbackExports(); } void toolDispatcher.initialize().catch((err) => { logger.error({ err }, "Failed to initialize plugin tool dispatcher"); @@ -435,13 +462,19 @@ export async function createApp( }).catch((err) => { logger.error({ err }, "Failed to load ready plugins on startup"); }); - process.once("exit", () => { - if (feedbackExportTimer) clearInterval(feedbackExportTimer); + let appServicesShutdown = false; + const shutdownAppServices = () => { + if (appServicesShutdown) return; + appServicesShutdown = true; + disableFeedbackExportFlushes(); devWatcher?.close(); viteHtmlRenderer?.dispose(); hostServiceCleanup.disposeAll(); hostServiceCleanup.teardown(); - }); + }; + app.locals.paperclipShutdown = shutdownAppServices; + + process.once("exit", shutdownAppServices); process.once("beforeExit", () => { void flushPluginLogBuffer(); }); diff --git a/server/src/index.ts b/server/src/index.ts index dd831c3bc62..c039f7584f1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -904,6 +904,9 @@ export async function startServer(): Promise { await telemetryClient.flush(); } + const appShutdown = (app as { locals?: { paperclipShutdown?: () => void } }).locals?.paperclipShutdown; + appShutdown?.(); + if (embeddedPostgres && embeddedPostgresStartedByThisProcess) { logger.info({ signal }, "Stopping embedded PostgreSQL"); try {