cjovs commited on
Commit
c5cd3f1
·
verified ·
1 Parent(s): b5c17a2

Back off plugin job scheduler when DB is down

Browse files

Pause plugin job scheduler ticks after database connection-refused errors to avoid repeated failed polling while the database is unavailable.

server/src/services/plugin-job-scheduler.ts CHANGED
@@ -38,6 +38,7 @@ import { and, eq, lte, or } from "drizzle-orm";
38
  import type { Db } from "@penclipai/db";
39
  import { pluginJobs, pluginJobRuns } from "@penclipai/db";
40
  import type { PluginJobStore } from "./plugin-job-store.js";
 
41
  import type { PluginWorkerManager } from "./plugin-worker-manager.js";
42
  import { parseCron, nextCronTick, validateCron } from "./cron.js";
43
  import { logger } from "../middleware/logger.js";
@@ -213,6 +214,11 @@ export function createPluginJobScheduler(
213
  } = options;
214
 
215
  const log = logger.child({ service: "plugin-job-scheduler" });
 
 
 
 
 
216
 
217
  // -----------------------------------------------------------------------
218
  // State
@@ -244,6 +250,10 @@ export function createPluginJobScheduler(
244
  * A single scheduler tick. Queries for due jobs and dispatches them.
245
  */
246
  async function tick(): Promise<void> {
 
 
 
 
247
  // Prevent overlapping ticks (in case a tick takes longer than the interval)
248
  if (tickInProgress) {
249
  log.debug("skipping tick — previous tick still in progress");
@@ -322,7 +332,16 @@ export function createPluginJobScheduler(
322
  if (dispatches.length > 0) {
323
  await Promise.allSettled(dispatches);
324
  }
 
325
  } catch (err) {
 
 
 
 
 
 
 
 
326
  log.error(
327
  { err: err instanceof Error ? err.message : String(err) },
328
  "scheduler tick error",
 
38
  import type { Db } from "@penclipai/db";
39
  import { pluginJobs, pluginJobRuns } from "@penclipai/db";
40
  import type { PluginJobStore } from "./plugin-job-store.js";
41
+ import { createDatabaseUnavailableGuard } from "../database-unavailable-guard.js";
42
  import type { PluginWorkerManager } from "./plugin-worker-manager.js";
43
  import { parseCron, nextCronTick, validateCron } from "./cron.js";
44
  import { logger } from "../middleware/logger.js";
 
214
  } = options;
215
 
216
  const log = logger.child({ service: "plugin-job-scheduler" });
217
+ const databaseGuard = createDatabaseUnavailableGuard({
218
+ service: "plugin-job-scheduler",
219
+ cooldownMs: Math.max(tickIntervalMs * 4, 60_000),
220
+ logger: log,
221
+ });
222
 
223
  // -----------------------------------------------------------------------
224
  // State
 
250
  * A single scheduler tick. Queries for due jobs and dispatches them.
251
  */
252
  async function tick(): Promise<void> {
253
+ if (!databaseGuard.shouldRun()) {
254
+ return;
255
+ }
256
+
257
  // Prevent overlapping ticks (in case a tick takes longer than the interval)
258
  if (tickInProgress) {
259
  log.debug("skipping tick — previous tick still in progress");
 
332
  if (dispatches.length > 0) {
333
  await Promise.allSettled(dispatches);
334
  }
335
+ databaseGuard.recordSuccess();
336
  } catch (err) {
337
+ if (
338
+ databaseGuard.handleError(
339
+ err,
340
+ "Pausing plugin job scheduler ticks because the database is currently unavailable",
341
+ )
342
+ ) {
343
+ return;
344
+ }
345
  log.error(
346
  { err: err instanceof Error ? err.message : String(err) },
347
  "scheduler tick error",