Back off plugin job scheduler when DB is down
Browse filesPause plugin job scheduler ticks after database connection-refused errors to avoid repeated failed polling while the database is unavailable.
server/src/services/plugin-job-scheduler.ts
CHANGED
|
@@ -38,6 +38,7 @@ import { and, eq, lte, or } from "drizzle-orm";
|
|
| 38 |
import type { Db } from "@penclipai/db";
|
| 39 |
import { pluginJobs, pluginJobRuns } from "@penclipai/db";
|
| 40 |
import type { PluginJobStore } from "./plugin-job-store.js";
|
|
|
|
| 41 |
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
| 42 |
import { parseCron, nextCronTick, validateCron } from "./cron.js";
|
| 43 |
import { logger } from "../middleware/logger.js";
|
|
@@ -213,6 +214,11 @@ export function createPluginJobScheduler(
|
|
| 213 |
} = options;
|
| 214 |
|
| 215 |
const log = logger.child({ service: "plugin-job-scheduler" });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
// -----------------------------------------------------------------------
|
| 218 |
// State
|
|
@@ -244,6 +250,10 @@ export function createPluginJobScheduler(
|
|
| 244 |
* A single scheduler tick. Queries for due jobs and dispatches them.
|
| 245 |
*/
|
| 246 |
async function tick(): Promise<void> {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
// Prevent overlapping ticks (in case a tick takes longer than the interval)
|
| 248 |
if (tickInProgress) {
|
| 249 |
log.debug("skipping tick — previous tick still in progress");
|
|
@@ -322,7 +332,16 @@ export function createPluginJobScheduler(
|
|
| 322 |
if (dispatches.length > 0) {
|
| 323 |
await Promise.allSettled(dispatches);
|
| 324 |
}
|
|
|
|
| 325 |
} catch (err) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
log.error(
|
| 327 |
{ err: err instanceof Error ? err.message : String(err) },
|
| 328 |
"scheduler tick error",
|
|
|
|
| 38 |
import type { Db } from "@penclipai/db";
|
| 39 |
import { pluginJobs, pluginJobRuns } from "@penclipai/db";
|
| 40 |
import type { PluginJobStore } from "./plugin-job-store.js";
|
| 41 |
+
import { createDatabaseUnavailableGuard } from "../database-unavailable-guard.js";
|
| 42 |
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
| 43 |
import { parseCron, nextCronTick, validateCron } from "./cron.js";
|
| 44 |
import { logger } from "../middleware/logger.js";
|
|
|
|
| 214 |
} = options;
|
| 215 |
|
| 216 |
const log = logger.child({ service: "plugin-job-scheduler" });
|
| 217 |
+
const databaseGuard = createDatabaseUnavailableGuard({
|
| 218 |
+
service: "plugin-job-scheduler",
|
| 219 |
+
cooldownMs: Math.max(tickIntervalMs * 4, 60_000),
|
| 220 |
+
logger: log,
|
| 221 |
+
});
|
| 222 |
|
| 223 |
// -----------------------------------------------------------------------
|
| 224 |
// State
|
|
|
|
| 250 |
* A single scheduler tick. Queries for due jobs and dispatches them.
|
| 251 |
*/
|
| 252 |
async function tick(): Promise<void> {
|
| 253 |
+
if (!databaseGuard.shouldRun()) {
|
| 254 |
+
return;
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
// Prevent overlapping ticks (in case a tick takes longer than the interval)
|
| 258 |
if (tickInProgress) {
|
| 259 |
log.debug("skipping tick — previous tick still in progress");
|
|
|
|
| 332 |
if (dispatches.length > 0) {
|
| 333 |
await Promise.allSettled(dispatches);
|
| 334 |
}
|
| 335 |
+
databaseGuard.recordSuccess();
|
| 336 |
} catch (err) {
|
| 337 |
+
if (
|
| 338 |
+
databaseGuard.handleError(
|
| 339 |
+
err,
|
| 340 |
+
"Pausing plugin job scheduler ticks because the database is currently unavailable",
|
| 341 |
+
)
|
| 342 |
+
) {
|
| 343 |
+
return;
|
| 344 |
+
}
|
| 345 |
log.error(
|
| 346 |
{ err: err instanceof Error ? err.message : String(err) },
|
| 347 |
"scheduler tick error",
|