light-infer-chat commited on
Commit
947ea10
·
1 Parent(s): 5d6260a

feat: cron job api added

Browse files
.gitignore CHANGED
@@ -134,4 +134,6 @@ tests
134
  deploy_hf.py
135
  .mimocode
136
 
137
- ddl
 
 
 
134
  deploy_hf.py
135
  .mimocode
136
 
137
+ ddl
138
+ API_DESCRIPTION.md
139
+ ENTERPRISE-API-ROADMAP*.md
app/api/deps.py CHANGED
@@ -13,6 +13,7 @@ from app.services.extraction_service import ExtractionService
13
  from app.services.ocr_service import OCRService
14
  from app.services.sql_validator_service import SqlValidatorService
15
  from app.services.text_cleaner_service import TextCleanerService
 
16
  from app.services.vector_store_service import VectorStoreService
17
 
18
 
@@ -54,6 +55,11 @@ def get_vector_store_service() -> VectorStoreService:
54
  return _vector_store_service
55
 
56
 
 
 
 
 
 
57
  def require_auth(token: str = Depends(require_api_key)) -> str:
58
  return token
59
 
 
13
  from app.services.ocr_service import OCRService
14
  from app.services.sql_validator_service import SqlValidatorService
15
  from app.services.text_cleaner_service import TextCleanerService
16
+ from app.services.scheduler_service import SchedulerService
17
  from app.services.vector_store_service import VectorStoreService
18
 
19
 
 
55
  return _vector_store_service
56
 
57
 
58
+ def get_scheduler_service() -> SchedulerService:
59
+ from app.api.server import _scheduler_service
60
+ return _scheduler_service
61
+
62
+
63
  def require_auth(token: str = Depends(require_api_key)) -> str:
64
  return token
65
 
app/api/server.py CHANGED
@@ -16,6 +16,7 @@ from app.core.logger import get_logger
16
  from app.core.redis_client import close_redis, create_redis_client
17
  from app.core.scripts import load_scripts
18
  from app.services.embeddings_service import EmbeddingService
 
19
  from app.services.vector_store_service import VectorStoreService
20
 
21
  _logger = get_logger(__name__)
@@ -23,6 +24,7 @@ _settings = get_settings()
23
 
24
  _embedding_service: EmbeddingService = EmbeddingService()
25
  _vector_store_service: VectorStoreService = VectorStoreService(_embedding_service)
 
26
 
27
 
28
  async def _self_ping():
@@ -78,8 +80,13 @@ async def lifespan(app: FastAPI):
78
  _logger.warning("Redis not configured, running in degraded mode")
79
 
80
  asyncio.create_task(_self_ping())
 
 
 
 
81
  yield
82
  _logger.info("Shutting down...")
 
83
  await close_redis(redis)
84
  await _vector_store_service.close_all()
85
  await pool_manager.close_all()
 
16
  from app.core.redis_client import close_redis, create_redis_client
17
  from app.core.scripts import load_scripts
18
  from app.services.embeddings_service import EmbeddingService
19
+ from app.services.scheduler_service import SchedulerService
20
  from app.services.vector_store_service import VectorStoreService
21
 
22
  _logger = get_logger(__name__)
 
24
 
25
  _embedding_service: EmbeddingService = EmbeddingService()
26
  _vector_store_service: VectorStoreService = VectorStoreService(_embedding_service)
27
+ _scheduler_service: SchedulerService = SchedulerService()
28
 
29
 
30
  async def _self_ping():
 
80
  _logger.warning("Redis not configured, running in degraded mode")
81
 
82
  asyncio.create_task(_self_ping())
83
+
84
+ await _scheduler_service.start()
85
+ _logger.info("Scheduler service started")
86
+
87
  yield
88
  _logger.info("Shutting down...")
89
+ await _scheduler_service.shutdown()
90
  await close_redis(redis)
91
  await _vector_store_service.close_all()
92
  await pool_manager.close_all()
app/api/v1/router.py CHANGED
@@ -16,6 +16,7 @@ from app.api.v1 import (
16
  qr_decoder,
17
  qr_generator,
18
  reconcile,
 
19
  scraper,
20
  semantic_router,
21
  sql_validator,
@@ -39,6 +40,7 @@ api_v1_router.include_router(embeddings.router, tags=["Embeddings"])
39
  api_v1_router.include_router(code_executor.router, tags=["Code Executor"])
40
  api_v1_router.include_router(verify_router, prefix="/verify", tags=["Verify"])
41
  api_v1_router.include_router(reconcile.router, tags=["Reconcile"])
 
42
  api_v1_router.include_router(scraper.router, tags=["Web Scraping"])
43
  api_v1_router.include_router(web_search.router, tags=["Web Search"])
44
  api_v1_router.include_router(sql_validator.router, tags=["SQL Validator"])
 
16
  qr_decoder,
17
  qr_generator,
18
  reconcile,
19
+ scheduler,
20
  scraper,
21
  semantic_router,
22
  sql_validator,
 
40
  api_v1_router.include_router(code_executor.router, tags=["Code Executor"])
41
  api_v1_router.include_router(verify_router, prefix="/verify", tags=["Verify"])
42
  api_v1_router.include_router(reconcile.router, tags=["Reconcile"])
43
+ api_v1_router.include_router(scheduler.router, tags=["Scheduler"])
44
  api_v1_router.include_router(scraper.router, tags=["Web Scraping"])
45
  api_v1_router.include_router(web_search.router, tags=["Web Search"])
46
  api_v1_router.include_router(sql_validator.router, tags=["SQL Validator"])
app/api/v1/scheduler.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ from fastapi import APIRouter, Depends, HTTPException, Query
6
+
7
+ from app.api.deps import get_scheduler_service, require_auth
8
+ from app.core.logger import get_logger
9
+ from app.services.scheduler_service import (
10
+ SchedulerService,
11
+ job_to_response,
12
+ history_to_response,
13
+ validate_cron_expression,
14
+ validate_timezone,
15
+ validate_url,
16
+ )
17
+
18
+ router = APIRouter()
19
+ logger = get_logger(__name__)
20
+
21
+
22
+ def _paginate(items: list[Any], total: int, page: int, page_size: int) -> dict[str, Any]:
23
+ return {
24
+ "items": items,
25
+ "total": total,
26
+ "page": page,
27
+ "page_size": page_size,
28
+ "has_next": (page * page_size) < total,
29
+ "has_prev": page > 1,
30
+ }
31
+
32
+
33
+ # ---------------------------------------------------------------------------
34
+ # Scheduler Monitoring
35
+ # ---------------------------------------------------------------------------
36
+
37
+ @router.get("/scheduler/status", summary="Get scheduler status")
38
+ async def get_scheduler_status(
39
+ token: str = Depends(require_auth),
40
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
41
+ ):
42
+ status = scheduler_service.get_scheduler_status()
43
+ status["running_job_ids"] = scheduler_service.get_running_job_ids()
44
+ return {"success": True, "data": status}
45
+
46
+
47
+ @router.get("/scheduler/metrics", summary="Get scheduler metrics")
48
+ async def get_scheduler_metrics(
49
+ token: str = Depends(require_auth),
50
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
51
+ ):
52
+ metrics = await scheduler_service.get_metrics()
53
+ return {"success": True, "data": metrics}
54
+
55
+
56
+ @router.get("/scheduler/health", summary="Get scheduler health (Redis + instance status)")
57
+ async def get_scheduler_health(
58
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
59
+ ):
60
+ health = await scheduler_service.get_health()
61
+ return {"success": True, "data": health}
62
+
63
+
64
+ # ---------------------------------------------------------------------------
65
+ # Job CRUD
66
+ # ---------------------------------------------------------------------------
67
+
68
+ @router.post("/scheduler/jobs", summary="Create a new scheduled job", status_code=201)
69
+ async def create_job(
70
+ body: dict[str, Any],
71
+ token: str = Depends(require_auth),
72
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
73
+ ):
74
+ errors = _validate_job_create(body)
75
+ if errors:
76
+ raise HTTPException(status_code=422, detail={"success": False, "errors": errors})
77
+
78
+ try:
79
+ job = await scheduler_service.create_job(body)
80
+ return {"success": True, "data": job_to_response(job)}
81
+ except ValueError as exc:
82
+ raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
83
+
84
+
85
+ @router.get("/scheduler/jobs", summary="List all scheduled jobs")
86
+ async def list_jobs(
87
+ status: str | None = Query(None, description="Filter by status (active, paused, completed, failed, deleted)"),
88
+ tags: str | None = Query(None, description="Comma-separated tags to filter by"),
89
+ page: int = Query(1, ge=1, description="Page number"),
90
+ page_size: int = Query(20, ge=1, le=100, description="Items per page"),
91
+ token: str = Depends(require_auth),
92
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
93
+ ):
94
+ tag_list = [t.strip() for t in tags.split(",") if t.strip()] if tags else None
95
+ jobs, total = await scheduler_service.list_jobs(
96
+ status=status, tags=tag_list, page=page, page_size=page_size,
97
+ )
98
+ return {
99
+ "success": True,
100
+ "data": _paginate(
101
+ items=[job_to_response(j) for j in jobs],
102
+ total=total, page=page, page_size=page_size,
103
+ ),
104
+ }
105
+
106
+
107
+ @router.get("/scheduler/jobs/{job_id}", summary="Get a scheduled job by ID")
108
+ async def get_job(
109
+ job_id: str,
110
+ token: str = Depends(require_auth),
111
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
112
+ ):
113
+ job = await scheduler_service.get_job(job_id)
114
+ if not job or job.get("status") == "deleted":
115
+ raise HTTPException(status_code=404, detail={"success": False, "error": "Job not found"})
116
+ return {"success": True, "data": job_to_response(job)}
117
+
118
+
119
+ @router.put("/scheduler/jobs/{job_id}", summary="Update a scheduled job")
120
+ async def update_job(
121
+ job_id: str,
122
+ body: dict[str, Any],
123
+ token: str = Depends(require_auth),
124
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
125
+ ):
126
+ try:
127
+ job = await scheduler_service.update_job(job_id, body)
128
+ return {"success": True, "data": job_to_response(job)}
129
+ except KeyError as exc:
130
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
131
+ except ValueError as exc:
132
+ raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
133
+
134
+
135
+ @router.delete("/scheduler/jobs/{job_id}", summary="Delete a scheduled job (soft delete)")
136
+ async def delete_job(
137
+ job_id: str,
138
+ token: str = Depends(require_auth),
139
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
140
+ ):
141
+ try:
142
+ await scheduler_service.delete_job(job_id)
143
+ return {"success": True, "message": "Job deleted successfully"}
144
+ except KeyError as exc:
145
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
146
+
147
+
148
+ @router.delete("/scheduler/jobs/{job_id}/hard", summary="Permanently delete a scheduled job")
149
+ async def hard_delete_job(
150
+ job_id: str,
151
+ token: str = Depends(require_auth),
152
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
153
+ ):
154
+ try:
155
+ await scheduler_service.hard_delete_job(job_id)
156
+ return {"success": True, "message": "Job permanently deleted"}
157
+ except KeyError as exc:
158
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
159
+
160
+
161
+ # ---------------------------------------------------------------------------
162
+ # Job Control
163
+ # ---------------------------------------------------------------------------
164
+
165
+ @router.post("/scheduler/jobs/{job_id}/pause", summary="Pause a scheduled job")
166
+ async def pause_job(
167
+ job_id: str,
168
+ token: str = Depends(require_auth),
169
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
170
+ ):
171
+ try:
172
+ job = await scheduler_service.pause_job(job_id)
173
+ return {"success": True, "data": job_to_response(job)}
174
+ except KeyError as exc:
175
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
176
+ except ValueError as exc:
177
+ raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
178
+
179
+
180
+ @router.post("/scheduler/jobs/{job_id}/resume", summary="Resume a paused job")
181
+ async def resume_job(
182
+ job_id: str,
183
+ token: str = Depends(require_auth),
184
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
185
+ ):
186
+ try:
187
+ job = await scheduler_service.resume_job(job_id)
188
+ return {"success": True, "data": job_to_response(job)}
189
+ except KeyError as exc:
190
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
191
+ except ValueError as exc:
192
+ raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
193
+
194
+
195
+ @router.post("/scheduler/jobs/{job_id}/run", summary="Trigger a job execution immediately")
196
+ async def run_job_now(
197
+ job_id: str,
198
+ token: str = Depends(require_auth),
199
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
200
+ ):
201
+ try:
202
+ await scheduler_service.run_job_now(job_id)
203
+ return {"success": True, "message": "Job execution triggered"}
204
+ except KeyError as exc:
205
+ raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
206
+ except ValueError as exc:
207
+ raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
208
+
209
+
210
+ # ---------------------------------------------------------------------------
211
+ # Execution History
212
+ # ---------------------------------------------------------------------------
213
+
214
+ @router.get("/scheduler/jobs/{job_id}/history", summary="Get execution history for a job")
215
+ async def get_job_history(
216
+ job_id: str,
217
+ page: int = Query(1, ge=1, description="Page number"),
218
+ page_size: int = Query(20, ge=1, le=100, description="Items per page"),
219
+ token: str = Depends(require_auth),
220
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
221
+ ):
222
+ job = await scheduler_service.get_job(job_id)
223
+ if not job:
224
+ raise HTTPException(status_code=404, detail={"success": False, "error": "Job not found"})
225
+
226
+ history, total = await scheduler_service.get_job_history(
227
+ job_id=job_id, page=page, page_size=page_size,
228
+ )
229
+ return {
230
+ "success": True,
231
+ "data": _paginate(
232
+ items=[history_to_response(h) for h in history],
233
+ total=total, page=page, page_size=page_size,
234
+ ),
235
+ }
236
+
237
+
238
+ @router.get("/scheduler/history", summary="Get global execution history")
239
+ async def get_execution_history(
240
+ page: int = Query(1, ge=1, description="Page number"),
241
+ page_size: int = Query(50, ge=1, le=200, description="Items per page"),
242
+ status: str | None = Query(None, description="Filter by execution status"),
243
+ token: str = Depends(require_auth),
244
+ scheduler_service: SchedulerService = Depends(get_scheduler_service),
245
+ ):
246
+ history, total = await scheduler_service.get_execution_history(
247
+ page=page, page_size=page_size, status=status,
248
+ )
249
+ return {
250
+ "success": True,
251
+ "data": _paginate(
252
+ items=[history_to_response(h) for h in history],
253
+ total=total, page=page, page_size=page_size,
254
+ ),
255
+ }
256
+
257
+
258
+ # ---------------------------------------------------------------------------
259
+ # Validation
260
+ # ---------------------------------------------------------------------------
261
+
262
+ def _validate_job_create(body: dict[str, Any]) -> list[str]:
263
+ errors: list[str] = []
264
+
265
+ if not body.get("name"):
266
+ errors.append("name is required")
267
+ elif not isinstance(body["name"], str) or len(body["name"]) > 255:
268
+ errors.append("name must be a string between 1 and 255 characters")
269
+
270
+ if not body.get("url"):
271
+ errors.append("url is required")
272
+ elif not isinstance(body["url"], str):
273
+ errors.append("url must be a string")
274
+ else:
275
+ try:
276
+ validate_url(body["url"])
277
+ except ValueError as exc:
278
+ errors.append(str(exc))
279
+
280
+ trigger = body.get("trigger")
281
+ if not trigger:
282
+ errors.append("trigger is required")
283
+ elif not isinstance(trigger, dict):
284
+ errors.append("trigger must be an object")
285
+ else:
286
+ trigger_type = trigger.get("type")
287
+ if not trigger_type:
288
+ errors.append("trigger.type is required")
289
+ elif trigger_type not in ("cron", "interval", "date"):
290
+ errors.append("trigger.type must be one of: cron, interval, date")
291
+ elif trigger_type == "cron":
292
+ if not trigger.get("cron_expression"):
293
+ errors.append("trigger.cron_expression is required for cron trigger")
294
+ else:
295
+ try:
296
+ validate_cron_expression(trigger["cron_expression"])
297
+ except ValueError as exc:
298
+ errors.append(str(exc))
299
+
300
+ if body.get("timezone"):
301
+ try:
302
+ validate_timezone(body["timezone"])
303
+ except ValueError as exc:
304
+ errors.append(str(exc))
305
+
306
+ timeout = body.get("timeout")
307
+ if timeout is not None:
308
+ if not isinstance(timeout, (int, float)):
309
+ errors.append("timeout must be a number")
310
+ elif timeout < 1 or timeout > 300:
311
+ errors.append("timeout must be between 1 and 300")
312
+
313
+ method = body.get("method")
314
+ if method and method not in ("GET", "POST", "PUT", "PATCH", "DELETE"):
315
+ errors.append("method must be one of: GET, POST, PUT, PATCH, DELETE")
316
+
317
+ retry = body.get("retry")
318
+ if retry is not None:
319
+ if not isinstance(retry, dict):
320
+ errors.append("retry must be an object")
321
+ else:
322
+ max_retries = retry.get("max_retries")
323
+ if max_retries is not None and (not isinstance(max_retries, int) or max_retries < 0 or max_retries > 10):
324
+ errors.append("retry.max_retries must be between 0 and 10")
325
+
326
+ return errors
app/config.py CHANGED
@@ -3,6 +3,7 @@ from __future__ import annotations
3
  from functools import lru_cache
4
  from typing import Optional
5
 
 
6
  from pydantic_settings import BaseSettings, SettingsConfigDict
7
 
8
 
@@ -83,6 +84,27 @@ class Settings(BaseSettings):
83
  lockout_minutes: int = 15
84
  admin_password: str = ""
85
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  @property
87
  def max_upload_mb(self) -> int:
88
  return self.max_upload_bytes // (1024 * 1024)
 
3
  from functools import lru_cache
4
  from typing import Optional
5
 
6
+ from pydantic import Field
7
  from pydantic_settings import BaseSettings, SettingsConfigDict
8
 
9
 
 
84
  lockout_minutes: int = 15
85
  admin_password: str = ""
86
 
87
+ # Redis connection
88
+ redis_host: str = Field(default="localhost", alias="REDIS_HOST")
89
+ redis_port: int = Field(default=6379, alias="REDIS_PORT")
90
+ redis_db: int = Field(default=0, alias="REDIS_DB")
91
+ redis_password: str | None = Field(default=None, alias="REDIS_PASSWORD")
92
+ redis_ssl: bool = Field(default=False, alias="REDIS_SSL")
93
+ redis_socket_timeout: int = Field(default=30, alias="REDIS_SOCKET_TIMEOUT")
94
+ redis_socket_connect_timeout: int = Field(default=30, alias="REDIS_SOCKET_CONNECT_TIMEOUT")
95
+ redis_retry_on_timeout: bool = Field(default=True, alias="REDIS_RETRY_ON_TIMEOUT")
96
+ redis_health_check_interval: int = Field(default=30, alias="REDIS_HEALTH_CHECK_INTERVAL")
97
+
98
+ # Scheduler settings
99
+ max_http_timeout: float = 300.0
100
+ default_scheduler_timezone: str = "UTC"
101
+ history_retention_days: int = 30
102
+ ssrf_protection: bool = True
103
+ scheduler_instance_id: str | None = Field(default=None, alias="SCHEDULER_INSTANCE_ID")
104
+ scheduler_lock_timeout: int = Field(default=300, alias="SCHEDULER_LOCK_TIMEOUT")
105
+ scheduler_misfire_grace_time: int = Field(default=300, alias="SCHEDULER_MISFIRE_GRACE_TIME")
106
+ scheduler_coordinator_prefix: str = Field(default="scheduler:", alias="SCHEDULER_COORDINATOR_PREFIX")
107
+
108
  @property
109
  def max_upload_mb(self) -> int:
110
  return self.max_upload_bytes // (1024 * 1024)
app/services/scheduler_service.py ADDED
@@ -0,0 +1,1358 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import hashlib
5
+ import json
6
+ import re
7
+ import secrets
8
+ import uuid
9
+ from datetime import UTC, datetime, timedelta
10
+ from enum import Enum
11
+ from typing import Any
12
+
13
+ import httpx
14
+ from apscheduler.executors.asyncio import AsyncIOExecutor
15
+ from apscheduler.jobstores.memory import MemoryJobStore
16
+ from apscheduler.schedulers.asyncio import AsyncIOScheduler
17
+ from apscheduler.triggers.cron import CronTrigger
18
+ from apscheduler.triggers.date import DateTrigger
19
+ from apscheduler.triggers.interval import IntervalTrigger
20
+ from croniter import croniter
21
+
22
+ from app.config import get_settings
23
+ from app.core.logger import get_logger
24
+ from app.services.supabase import SupabaseClient, get_supabase_client
25
+
26
+ logger = get_logger(__name__)
27
+ settings = get_settings()
28
+
29
+ # ---------------------------------------------------------------------------
30
+ # Redis lazy import helper
31
+ # ---------------------------------------------------------------------------
32
+
33
+ _redis_imported: bool = False
34
+ _redis_asyncio: Any = None
35
+
36
+
37
+ def _get_redis_asyncio():
38
+ global _redis_imported, _redis_asyncio
39
+ if not _redis_imported:
40
+ try:
41
+ import redis.asyncio as ra
42
+ _redis_asyncio = ra
43
+ except ImportError:
44
+ _redis_asyncio = None
45
+ _redis_imported = True
46
+ return _redis_asyncio
47
+
48
+
49
+ def _get_redis_jobstore_cls():
50
+ try:
51
+ from apscheduler.jobstores.redis import RedisJobStore
52
+ return RedisJobStore
53
+ except ImportError:
54
+ return None
55
+
56
+
57
+ # ---------------------------------------------------------------------------
58
+ # Enumerations
59
+ # ---------------------------------------------------------------------------
60
+
61
+ class JobStatus(str, Enum):
62
+ ACTIVE = "active"
63
+ PAUSED = "paused"
64
+ COMPLETED = "completed"
65
+ FAILED = "failed"
66
+ DELETED = "deleted"
67
+
68
+
69
+ class TriggerType(str, Enum):
70
+ CRON = "cron"
71
+ INTERVAL = "interval"
72
+ DATE = "date"
73
+
74
+
75
+ class HttpMethod(str, Enum):
76
+ GET = "GET"
77
+ POST = "POST"
78
+ PUT = "PUT"
79
+ PATCH = "PATCH"
80
+ DELETE = "DELETE"
81
+
82
+
83
+ class AuthType(str, Enum):
84
+ NONE = "none"
85
+ BEARER = "bearer"
86
+ API_KEY = "api_key"
87
+ BASIC = "basic"
88
+ CUSTOM = "custom"
89
+
90
+
91
+ class ExecutionStatus(str, Enum):
92
+ SUCCESS = "success"
93
+ FAILURE = "failure"
94
+ TIMEOUT = "timeout"
95
+
96
+
97
+ class TriggerReason(str, Enum):
98
+ SCHEDULED = "scheduled"
99
+ MANUAL = "manual"
100
+ RETRY = "retry"
101
+
102
+
103
+ # ---------------------------------------------------------------------------
104
+ # SSRF Protection
105
+ # ---------------------------------------------------------------------------
106
+
107
+ _PRIVATE_NETWORKS = [
108
+ re.compile(r"^10\."),
109
+ re.compile(r"^172\.(1[6-9]|2\d|3[01])\."),
110
+ re.compile(r"^192\.168\."),
111
+ re.compile(r"^127\."),
112
+ re.compile(r"^0\."),
113
+ re.compile(r"^169\.254\."),
114
+ re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\."),
115
+ re.compile(r"^fc[0-9a-f]{2}:", re.IGNORECASE),
116
+ re.compile(r"^fe80:", re.IGNORECASE),
117
+ ]
118
+
119
+ _BLOCKED_HOSTS: set[str] = {
120
+ "localhost", "127.0.0.1", "0.0.0.0", "::1",
121
+ "169.254.169.254",
122
+ "metadata.google.internal",
123
+ }
124
+
125
+
126
+ def _check_ssrf(url: str) -> None:
127
+ if not settings.ssrf_protection:
128
+ return
129
+ from urllib.parse import urlparse
130
+ parsed = urlparse(url)
131
+ host = parsed.hostname or ""
132
+ if host.lower() in _BLOCKED_HOSTS:
133
+ raise ValueError(f"SSRF protection: host '{host}' is blocked")
134
+ for pattern in _PRIVATE_NETWORKS:
135
+ if pattern.match(host):
136
+ raise ValueError(f"SSRF protection: host '{host}' resolves to a private network")
137
+
138
+
139
+ # ---------------------------------------------------------------------------
140
+ # Auth Header Builder
141
+ # ---------------------------------------------------------------------------
142
+
143
+ def _build_auth_headers(auth_config: dict[str, Any]) -> dict[str, str]:
144
+ auth_headers: dict[str, str] = {}
145
+ auth_type = auth_config.get("type", "none")
146
+ if auth_type == "bearer":
147
+ auth_headers["Authorization"] = f"Bearer {auth_config.get('token', '')}"
148
+ elif auth_type == "api_key":
149
+ header_name = auth_config.get("api_key_header", "X-API-Key")
150
+ auth_headers[header_name] = auth_config.get("api_key", "")
151
+ elif auth_type == "basic":
152
+ import base64
153
+ username = auth_config.get("username", "")
154
+ password = auth_config.get("password", "")
155
+ encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
156
+ auth_headers["Authorization"] = f"Basic {encoded}"
157
+ elif auth_type == "custom":
158
+ custom = auth_config.get("custom_headers") or {}
159
+ auth_headers.update(custom)
160
+ return auth_headers
161
+
162
+
163
+ # ---------------------------------------------------------------------------
164
+ # Scheduler Repository (Supabase-backed)
165
+ # ---------------------------------------------------------------------------
166
+
167
+ class SchedulerRepository:
168
+ def __init__(self, client: SupabaseClient):
169
+ self._client = client
170
+
171
+ async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
172
+ result = await self._client.insert("scheduled_jobs", data)
173
+ return result or {}
174
+
175
+ async def get_job_by_id(self, job_id: str) -> dict[str, Any] | None:
176
+ try:
177
+ return await self._client.find_one("scheduled_jobs", "id", job_id)
178
+ except Exception:
179
+ return None
180
+
181
+ async def get_job_by_name(self, name: str) -> dict[str, Any] | None:
182
+ try:
183
+ return await self._client.find_one("scheduled_jobs", "name", name)
184
+ except Exception:
185
+ return None
186
+
187
+ async def list_jobs(
188
+ self,
189
+ status: str | None = None,
190
+ tags: list[str] | None = None,
191
+ page: int = 1,
192
+ page_size: int = 20,
193
+ ) -> tuple[list[dict[str, Any]], int]:
194
+ offset = (page - 1) * page_size
195
+ if status and tags:
196
+ rows = await self._client.select("scheduled_jobs", eq=("status", status), limit=page_size, offset=offset)
197
+ filtered = [r for r in rows if tags and any(t in (r.get("tags") or []) for t in tags)]
198
+ total = len(filtered)
199
+ return filtered[:page_size], total
200
+ elif status:
201
+ rows = await self._client.select("scheduled_jobs", eq=("status", status), limit=page_size, offset=offset)
202
+ total = len(await self._client.select("scheduled_jobs", eq=("status", status)))
203
+ return rows, total
204
+ else:
205
+ rows = await self._client.select("scheduled_jobs", limit=page_size, offset=offset)
206
+ total = len(await self._client.select("scheduled_jobs"))
207
+ return rows, total
208
+
209
+ async def update_job(self, job_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
210
+ data["updated_at"] = datetime.now(UTC).isoformat()
211
+ result = await self._client.update("scheduled_jobs", "id", job_id, data)
212
+ return result[0] if result else None
213
+
214
+ async def delete_job(self, job_id: str) -> None:
215
+ await self._client.delete("scheduled_jobs", "id", job_id)
216
+
217
+ async def get_active_jobs(self) -> list[dict[str, Any]]:
218
+ rows = await self._client.select("scheduled_jobs", eq=("status", JobStatus.ACTIVE.value))
219
+ paused = await self._client.select("scheduled_jobs", eq=("status", JobStatus.PAUSED.value))
220
+ return rows + paused
221
+
222
+ async def create_history(self, data: dict[str, Any]) -> dict[str, Any]:
223
+ result = await self._client.insert("job_execution_history", data)
224
+ return result or {}
225
+
226
+ async def get_history(
227
+ self,
228
+ job_id: str,
229
+ page: int = 1,
230
+ page_size: int = 20,
231
+ ) -> tuple[list[dict[str, Any]], int]:
232
+ offset = (page - 1) * page_size
233
+ rows = await self._client.select(
234
+ "job_execution_history", eq=("job_id", job_id),
235
+ order=("started_at", True),
236
+ limit=page_size, offset=offset,
237
+ )
238
+ total = await self._client.count("job_execution_history", "job_id", job_id)
239
+ return rows, total
240
+
241
+ async def get_history_all(
242
+ self,
243
+ page: int = 1,
244
+ page_size: int = 50,
245
+ status_filter: str | None = None,
246
+ ) -> tuple[list[dict[str, Any]], int]:
247
+ offset = (page - 1) * page_size
248
+ if status_filter:
249
+ rows = await self._client.select(
250
+ "job_execution_history", eq=("status", status_filter),
251
+ order=("started_at", True),
252
+ limit=page_size, offset=offset,
253
+ )
254
+ total = await self._client.count("job_execution_history", "status", status_filter)
255
+ else:
256
+ rows = await self._client.select(
257
+ "job_execution_history",
258
+ order=("started_at", True),
259
+ limit=page_size, offset=offset,
260
+ )
261
+ total = len(await self._client.select("job_execution_history"))
262
+ return rows, total
263
+
264
+ async def purge_old_history(self, days: int) -> int:
265
+ cutoff = (datetime.now(UTC) - timedelta(days=days)).isoformat()
266
+ rows = await self._client.select("job_execution_history")
267
+ deleted = 0
268
+ for row in rows:
269
+ created = row.get("created_at", "")
270
+ if created < cutoff:
271
+ await self._client.delete("job_execution_history", "id", row["id"])
272
+ deleted += 1
273
+ return deleted
274
+
275
+ async def count_jobs(self, status: str | None = None) -> int:
276
+ if status:
277
+ return await self._client.count("scheduled_jobs", "status", status)
278
+ rows = await self._client.select("scheduled_jobs")
279
+ return len(rows)
280
+
281
+ async def count_executions(self, status: str | None = None) -> int:
282
+ if status:
283
+ return await self._client.count("job_execution_history", "status", status)
284
+ rows = await self._client.select("job_execution_history")
285
+ return len(rows)
286
+
287
+
288
+ # ---------------------------------------------------------------------------
289
+ # HTTP Execution Engine
290
+ # ---------------------------------------------------------------------------
291
+
292
+ class HttpExecutionResult:
293
+ __slots__ = (
294
+ "success", "status_code", "response_body",
295
+ "response_size", "duration_ms", "error_message",
296
+ "exception_type", "retry_count",
297
+ )
298
+
299
+ def __init__(
300
+ self,
301
+ success: bool,
302
+ status_code: int | None = None,
303
+ response_body: str | None = None,
304
+ response_size: int = 0,
305
+ duration_ms: float = 0.0,
306
+ error_message: str | None = None,
307
+ exception_type: str | None = None,
308
+ retry_count: int = 0,
309
+ ) -> None:
310
+ self.success = success
311
+ self.status_code = status_code
312
+ self.response_body = response_body
313
+ self.response_size = response_size
314
+ self.duration_ms = duration_ms
315
+ self.error_message = error_message
316
+ self.exception_type = exception_type
317
+ self.retry_count = retry_count
318
+
319
+
320
+ class HttpExecutionEngine:
321
+ _MAX_RESPONSE_PREVIEW = 500
322
+
323
+ def __init__(self) -> None:
324
+ self._client: httpx.AsyncClient | None = None
325
+ self._lock = asyncio.Lock()
326
+
327
+ async def _get_client(self) -> httpx.AsyncClient:
328
+ if self._client is None or self._client.is_closed:
329
+ async with self._lock:
330
+ if self._client is None or self._client.is_closed:
331
+ self._client = httpx.AsyncClient(
332
+ follow_redirects=True,
333
+ limits=httpx.Limits(
334
+ max_connections=200,
335
+ max_keepalive_connections=50,
336
+ keepalive_expiry=30,
337
+ ),
338
+ timeout=httpx.Timeout(settings.max_http_timeout),
339
+ )
340
+ return self._client
341
+
342
+ async def close(self) -> None:
343
+ if self._client and not self._client.is_closed:
344
+ await self._client.aclose()
345
+ logger.info("HTTP client closed")
346
+
347
+ async def execute(self, job: dict[str, Any]) -> HttpExecutionResult:
348
+ retry_on_status: list[int] = job.get("retry_on_status") or [429, 500, 502, 503, 504]
349
+ max_retries: int = job.get("max_retries", 3)
350
+ retry_delay: float = job.get("retry_delay_seconds", 1.0)
351
+ backoff: float = job.get("retry_backoff", 2.0)
352
+ max_delay: float = job.get("retry_max_delay_seconds", 60.0)
353
+ retry_jitter: bool = job.get("retry_jitter", False)
354
+
355
+ last_result: HttpExecutionResult | None = None
356
+
357
+ for attempt in range(max_retries + 1):
358
+ if attempt > 0:
359
+ delay = min(retry_delay * (backoff ** (attempt - 1)), max_delay)
360
+ if retry_jitter:
361
+ delay = delay * (0.5 + secrets.randbelow(1000) / 1000.0)
362
+ logger.info("Job retry wait", extra={
363
+ "job_id": job["id"], "job_name": job["name"],
364
+ "attempt": attempt, "delay_seconds": round(delay, 2),
365
+ })
366
+ await asyncio.sleep(delay)
367
+
368
+ result = await self._execute_once(job, attempt)
369
+
370
+ if result.success:
371
+ result.retry_count = attempt
372
+ return result
373
+
374
+ last_result = result
375
+
376
+ should_retry = False
377
+ if attempt < max_retries:
378
+ if result.exception_type == "TimeoutException" and job.get("retry_on_timeout", True):
379
+ should_retry = True
380
+ elif result.exception_type in ("ConnectError", "ReadError", "NetworkError") and job.get("retry_on_failure", True):
381
+ should_retry = True
382
+ elif result.status_code in retry_on_status:
383
+ should_retry = True
384
+
385
+ if not should_retry:
386
+ break
387
+
388
+ logger.warning("Job execution retry", extra={
389
+ "job_id": job["id"], "job_name": job["name"],
390
+ "attempt": attempt + 1, "max_retries": max_retries,
391
+ "status_code": result.status_code, "error": result.error_message,
392
+ })
393
+
394
+ if last_result is not None:
395
+ last_result.retry_count = max(0, max_retries)
396
+ return last_result or HttpExecutionResult(success=False, error_message="Unknown error")
397
+
398
+ async def _execute_once(self, job: dict[str, Any], attempt: int) -> HttpExecutionResult:
399
+ started = datetime.now(UTC)
400
+ client = await self._get_client()
401
+
402
+ headers: dict[str, str] = dict(job.get("headers") or {})
403
+ auth_headers = _build_auth_headers(job.get("auth_config") or {})
404
+ headers.update(auth_headers)
405
+ headers.setdefault("User-Agent", f"{settings.app_name}/{settings.app_version}")
406
+
407
+ params: dict[str, str] = dict(job.get("query_params") or {})
408
+ method = (job.get("method") or "GET").upper()
409
+ url = job.get("url", "")
410
+
411
+ try:
412
+ _check_ssrf(url)
413
+ except ValueError as exc:
414
+ return HttpExecutionResult(
415
+ success=False, error_message=str(exc), exception_type="SSRFViolation",
416
+ )
417
+
418
+ request_kwargs: dict[str, Any] = {
419
+ "method": method,
420
+ "url": url,
421
+ "headers": headers,
422
+ "params": params,
423
+ "timeout": job.get("timeout_seconds", 30.0),
424
+ }
425
+
426
+ body = job.get("body")
427
+ if body is not None and method in ("POST", "PUT", "PATCH", "DELETE"):
428
+ body_type = job.get("body_type", "json")
429
+ if body_type == "json":
430
+ request_kwargs["json"] = body
431
+ elif body_type == "form":
432
+ request_kwargs["data"] = body
433
+ elif body_type == "raw":
434
+ request_kwargs["content"] = str(body).encode()
435
+
436
+ try:
437
+ response = await client.request(**request_kwargs)
438
+ duration_ms = (datetime.now(UTC) - started).total_seconds() * 1000
439
+ response_body = response.text[:self._MAX_RESPONSE_PREVIEW]
440
+ success = response.is_success
441
+
442
+ if not success:
443
+ logger.warning("HTTP non-success status", extra={
444
+ "job_id": job["id"], "job_name": job["name"],
445
+ "status_code": response.status_code, "attempt": attempt,
446
+ })
447
+
448
+ return HttpExecutionResult(
449
+ success=success,
450
+ status_code=response.status_code,
451
+ response_body=response_body,
452
+ response_size=len(response.content),
453
+ duration_ms=duration_ms,
454
+ error_message=None if success else f"HTTP {response.status_code}",
455
+ )
456
+
457
+ except httpx.TimeoutException as exc:
458
+ duration_ms = (datetime.now(UTC) - started).total_seconds() * 1000
459
+ return HttpExecutionResult(
460
+ success=False, duration_ms=duration_ms,
461
+ error_message=f"Request timed out after {job.get('timeout_seconds', 30.0)}s",
462
+ exception_type="TimeoutException",
463
+ )
464
+ except httpx.ConnectError as exc:
465
+ return HttpExecutionResult(
466
+ success=False, error_message=f"Connection error: {exc}",
467
+ exception_type="ConnectError",
468
+ )
469
+ except httpx.ReadError as exc:
470
+ return HttpExecutionResult(
471
+ success=False, error_message=f"Read error: {exc}",
472
+ exception_type="ReadError",
473
+ )
474
+ except Exception as exc:
475
+ logger.error("HTTP unexpected error", extra={
476
+ "job_id": job["id"], "job_name": job["name"],
477
+ "attempt": attempt, "error": str(exc),
478
+ })
479
+ return HttpExecutionResult(
480
+ success=False, error_message=str(exc),
481
+ exception_type=type(exc).__name__,
482
+ )
483
+
484
+
485
+ # ---------------------------------------------------------------------------
486
+ # Scheduler Service — Production-Grade with Redis HA
487
+ # ---------------------------------------------------------------------------
488
+
489
+ class SchedulerService:
490
+ def __init__(self) -> None:
491
+ self._scheduler: AsyncIOScheduler | None = None
492
+ self._async_redis: Any = None
493
+ self._use_redis = False
494
+ self._instance_id: str = settings.scheduler_instance_id or str(uuid.uuid4())[:8]
495
+ self._http_engine = HttpExecutionEngine()
496
+ self._running_jobs: dict[str, asyncio.Task[None]] = {}
497
+ self._lock = asyncio.Lock()
498
+ self._maintenance_task: asyncio.Task[None] | None = None
499
+ self._repo: SchedulerRepository | None = None
500
+
501
+ # -----------------------------------------------------------------------
502
+ # Redis connection management
503
+ # -----------------------------------------------------------------------
504
+
505
+ async def _connect_async_redis(self) -> Any:
506
+ ra = _get_redis_asyncio()
507
+ if ra is None:
508
+ return None
509
+ try:
510
+ client = ra.Redis(
511
+ host=settings.redis_host,
512
+ port=settings.redis_port,
513
+ password=settings.redis_password or None,
514
+ db=settings.redis_db,
515
+ ssl=settings.redis_ssl,
516
+ socket_timeout=settings.redis_socket_timeout,
517
+ socket_connect_timeout=settings.redis_socket_connect_timeout,
518
+ retry_on_timeout=settings.redis_retry_on_timeout,
519
+ health_check_interval=settings.redis_health_check_interval,
520
+ decode_responses=True,
521
+ )
522
+ await client.ping()
523
+ logger.info(
524
+ "Connected to Redis at %s:%d (db=%d)",
525
+ settings.redis_host, settings.redis_port, settings.redis_db,
526
+ )
527
+ return client
528
+ except Exception as exc:
529
+ logger.warning("Redis connection failed: %s — falling back to in-memory scheduler", exc)
530
+ return None
531
+
532
+ def _create_redis_jobstore(self) -> Any:
533
+ cls = _get_redis_jobstore_cls()
534
+ if cls is None:
535
+ raise RuntimeError("apscheduler[jobstores_redis] not installed")
536
+ return cls(
537
+ db=settings.redis_db,
538
+ host=settings.redis_host,
539
+ port=settings.redis_port,
540
+ password=settings.redis_password or None,
541
+ ssl=settings.redis_ssl,
542
+ socket_timeout=settings.redis_socket_timeout,
543
+ socket_connect_timeout=settings.redis_socket_connect_timeout,
544
+ retry_on_timeout=settings.redis_retry_on_timeout,
545
+ health_check_interval=settings.redis_health_check_interval,
546
+ )
547
+
548
+ # -----------------------------------------------------------------------
549
+ # Distributed execution lock (Redis SETNX)
550
+ # -----------------------------------------------------------------------
551
+
552
+ def _lock_key(self, job_id: str) -> str:
553
+ return f"{settings.scheduler_coordinator_prefix}lock:{job_id}"
554
+
555
+ async def _acquire_execution_lock(self, job: dict[str, Any]) -> bool:
556
+ if not self._use_redis or self._async_redis is None:
557
+ return True
558
+ key = self._lock_key(job["id"])
559
+ ttl = job.get("timeout_seconds", 30) + 30
560
+ acquired = await self._async_redis.setnx(key, self._instance_id)
561
+ if acquired:
562
+ await self._async_redis.expire(key, int(ttl))
563
+ logger.debug("Acquired execution lock for job %s", job["id"])
564
+ return True
565
+ owner = await self._async_redis.get(key)
566
+ logger.warning(
567
+ "Execution lock held by instance %s for job %s — skipping",
568
+ owner, job["id"],
569
+ )
570
+ return False
571
+
572
+ async def _release_execution_lock(self, job_id: str) -> None:
573
+ if not self._use_redis or self._async_redis is None:
574
+ return
575
+ key = self._lock_key(job_id)
576
+ await self._async_redis.delete(key)
577
+ logger.debug("Released execution lock for job %s", job_id)
578
+
579
+ # -----------------------------------------------------------------------
580
+ # Redis health
581
+ # -----------------------------------------------------------------------
582
+
583
+ async def _check_redis_health(self) -> dict[str, Any]:
584
+ if not self._use_redis or self._async_redis is None:
585
+ return {"connected": False, "mode": "memory"}
586
+ try:
587
+ ping = await self._async_redis.ping()
588
+ info = await self._async_redis.info(section="server")
589
+ return {
590
+ "connected": bool(ping),
591
+ "mode": "redis",
592
+ "redis_version": info.get("redis_version", "unknown"),
593
+ "instance_id": self._instance_id,
594
+ }
595
+ except Exception as exc:
596
+ return {"connected": False, "mode": "redis", "error": str(exc)}
597
+
598
+ # -----------------------------------------------------------------------
599
+ # Lifecycle
600
+ # -----------------------------------------------------------------------
601
+
602
+ async def start(self) -> None:
603
+ self._async_redis = await self._connect_async_redis()
604
+ self._use_redis = self._async_redis is not None
605
+
606
+ jobstores: dict[str, Any] = {}
607
+ if self._use_redis:
608
+ try:
609
+ jobstores["default"] = self._create_redis_jobstore()
610
+ logger.info("Using RedisJobStore for job persistence")
611
+ except Exception as exc:
612
+ logger.warning("Failed to create RedisJobStore: %s — falling back to memory", exc)
613
+ self._use_redis = False
614
+ self._async_redis = None
615
+
616
+ if not self._use_redis:
617
+ jobstores["default"] = MemoryJobStore()
618
+ logger.info("Using MemoryJobStore (jobs will not survive restart)")
619
+
620
+ executors = {"default": AsyncIOExecutor()}
621
+ job_defaults = {
622
+ "coalesce": True,
623
+ "max_instances": 1,
624
+ "misfire_grace_time": settings.scheduler_misfire_grace_time,
625
+ }
626
+
627
+ self._scheduler = AsyncIOScheduler(
628
+ jobstores=jobstores,
629
+ executors=executors,
630
+ job_defaults=job_defaults,
631
+ )
632
+ self._scheduler.start()
633
+ logger.info("Scheduler started (mode=%s, instance=%s)", "redis" if self._use_redis else "memory", self._instance_id)
634
+
635
+ try:
636
+ client = get_supabase_client()
637
+ if client:
638
+ repo = SchedulerRepository(client)
639
+ jobs = await repo.get_active_jobs()
640
+ restored = 0
641
+ for job in jobs:
642
+ try:
643
+ self._schedule_job(job)
644
+ restored += 1
645
+ except Exception as exc:
646
+ logger.error("Failed to restore job %s: %s", job.get("id"), exc)
647
+
648
+ if self._use_redis:
649
+ aps_jobs = self._scheduler.get_jobs()
650
+ aps_ids = {j.id for j in aps_jobs}
651
+ db_ids = {f"job_{j['id']}" for j in jobs}
652
+ stale = aps_ids - db_ids
653
+ for sid in stale:
654
+ try:
655
+ self._scheduler.remove_job(sid)
656
+ logger.info("Removed stale job %s from Redis store", sid)
657
+ except Exception:
658
+ pass
659
+
660
+ logger.info("Restored %d jobs from database", restored)
661
+ except Exception as exc:
662
+ logger.warning("Could not restore jobs from database: %s", exc)
663
+
664
+ self._maintenance_task = asyncio.create_task(self._maintenance_loop())
665
+
666
+ async def shutdown(self) -> None:
667
+ logger.info("Scheduler shutting down (instance=%s)", self._instance_id)
668
+ if self._maintenance_task:
669
+ self._maintenance_task.cancel()
670
+ try:
671
+ await asyncio.wait_for(asyncio.shield(self._maintenance_task), timeout=5.0)
672
+ except (asyncio.CancelledError, asyncio.TimeoutError):
673
+ pass
674
+
675
+ if self._scheduler:
676
+ self._scheduler.shutdown(wait=False)
677
+
678
+ async with self._lock:
679
+ tasks = list(self._running_jobs.values())
680
+ for task in tasks:
681
+ task.cancel()
682
+ try:
683
+ await asyncio.wait_for(asyncio.shield(task), timeout=5.0)
684
+ except (asyncio.CancelledError, asyncio.TimeoutError):
685
+ pass
686
+
687
+ await self._http_engine.close()
688
+
689
+ if self._async_redis:
690
+ try:
691
+ await self._async_redis.aclose()
692
+ logger.info("Redis connection closed")
693
+ except Exception:
694
+ pass
695
+
696
+ logger.info("Scheduler shutdown complete")
697
+
698
+ async def _maintenance_loop(self) -> None:
699
+ while True:
700
+ try:
701
+ await asyncio.sleep(3600)
702
+ if self._use_redis and self._async_redis:
703
+ try:
704
+ await self._async_redis.ping()
705
+ except Exception:
706
+ logger.error("Redis ping failed in maintenance loop")
707
+ client = get_supabase_client()
708
+ if client:
709
+ repo = SchedulerRepository(client)
710
+ purged = await repo.purge_old_history(settings.history_retention_days)
711
+ if purged:
712
+ logger.info("Purged %d old history records", purged)
713
+ except asyncio.CancelledError:
714
+ break
715
+ except Exception:
716
+ logger.error("Maintenance loop error", exc_info=True)
717
+
718
+ # -----------------------------------------------------------------------
719
+ # APScheduler helpers
720
+ # -----------------------------------------------------------------------
721
+
722
+ def _build_trigger(self, job: dict[str, Any]):
723
+ tz = job.get("timezone", "UTC")
724
+ cfg = job.get("trigger_config") or {}
725
+ trigger_type = job.get("trigger_type")
726
+
727
+ if trigger_type == TriggerType.CRON.value:
728
+ cron_expr = cfg.get("cron_expression", "* * * * *")
729
+ parts = cron_expr.split()
730
+ return CronTrigger(
731
+ second=parts[0] if len(parts) > 0 else "*",
732
+ minute=parts[1] if len(parts) > 1 else "*",
733
+ hour=parts[2] if len(parts) > 2 else "*",
734
+ day=parts[3] if len(parts) > 3 else "*",
735
+ month=parts[4] if len(parts) > 4 else "*",
736
+ day_of_week=parts[5] if len(parts) > 5 else "*",
737
+ timezone=tz,
738
+ start_date=job.get("start_date"),
739
+ end_date=job.get("end_date"),
740
+ jitter=job.get("jitter_seconds"),
741
+ )
742
+ elif trigger_type == TriggerType.INTERVAL.value:
743
+ return IntervalTrigger(
744
+ weeks=cfg.get("weeks", 0),
745
+ days=cfg.get("days", 0),
746
+ hours=cfg.get("hours", 0),
747
+ minutes=cfg.get("minutes", 0),
748
+ seconds=cfg.get("seconds", 0),
749
+ timezone=tz,
750
+ start_date=job.get("start_date"),
751
+ end_date=job.get("end_date"),
752
+ jitter=job.get("jitter_seconds"),
753
+ )
754
+ elif trigger_type == TriggerType.DATE.value:
755
+ run_date = cfg.get("run_date")
756
+ return DateTrigger(run_date=run_date, timezone=tz)
757
+ raise ValueError(f"Unknown trigger type: {trigger_type}")
758
+
759
+ def _schedule_job(self, job: dict[str, Any]) -> None:
760
+ if self._scheduler is None:
761
+ raise RuntimeError("Scheduler not started")
762
+ trigger = self._build_trigger(job)
763
+ aps_id = f"job_{job['id']}"
764
+ self._scheduler.add_job(
765
+ func=self._execute_job_wrapper,
766
+ trigger=trigger,
767
+ args=[job["id"]],
768
+ id=aps_id,
769
+ name=job.get("name", "unknown"),
770
+ coalesce=job.get("coalesce", True),
771
+ max_instances=job.get("max_instances", 1),
772
+ misfire_grace_time=job.get("misfire_grace_time") or settings.scheduler_misfire_grace_time,
773
+ replace_existing=True,
774
+ )
775
+ logger.info("Scheduled job %s (%s)", job["name"], job["id"])
776
+
777
+ def _reschedule_job(self, job: dict[str, Any]) -> None:
778
+ aps_id = f"job_{job['id']}"
779
+ try:
780
+ self._scheduler.remove_job(aps_id)
781
+ except Exception:
782
+ pass
783
+ if job.get("status") in (JobStatus.ACTIVE.value, JobStatus.PAUSED.value):
784
+ self._schedule_job(job)
785
+ if job.get("status") == JobStatus.PAUSED.value:
786
+ try:
787
+ self._scheduler.pause_job(aps_id)
788
+ except Exception:
789
+ pass
790
+
791
+ def _remove_aps_job(self, job_id: str) -> None:
792
+ aps_id = f"job_{job_id}"
793
+ try:
794
+ self._scheduler.remove_job(aps_id)
795
+ except Exception:
796
+ pass
797
+
798
+ def pause_aps_job(self, job_id: str) -> None:
799
+ aps_id = f"job_{job_id}"
800
+ try:
801
+ self._scheduler.pause_job(aps_id)
802
+ except Exception as exc:
803
+ raise RuntimeError(f"Failed to pause job: {exc}") from exc
804
+
805
+ def resume_aps_job(self, job_id: str) -> None:
806
+ aps_id = f"job_{job_id}"
807
+ try:
808
+ self._scheduler.resume_job(aps_id)
809
+ except Exception as exc:
810
+ raise RuntimeError(f"Failed to resume job: {exc}") from exc
811
+
812
+ def get_next_run_time(self, job_id: str) -> datetime | None:
813
+ if self._scheduler is None:
814
+ return None
815
+ aps_id = f"job_{job_id}"
816
+ aps_job = self._scheduler.get_job(aps_id)
817
+ return aps_job.next_run_time if aps_job else None
818
+
819
+ def get_scheduler_status(self) -> dict[str, Any]:
820
+ if self._scheduler is None:
821
+ return {"running": False, "pending_jobs": 0, "currently_executing": 0}
822
+ return {
823
+ "running": self._scheduler.running,
824
+ "pending_jobs": len(self._scheduler.get_jobs()),
825
+ "currently_executing": len(self._running_jobs),
826
+ }
827
+
828
+ async def get_health(self) -> dict[str, Any]:
829
+ redis_status = await self._check_redis_health()
830
+ scheduler_status = self.get_scheduler_status()
831
+ return {
832
+ "scheduler": scheduler_status,
833
+ "redis": redis_status,
834
+ "instance_id": self._instance_id,
835
+ "mode": "redis" if self._use_redis else "memory",
836
+ "running_job_ids": self.get_running_job_ids(),
837
+ }
838
+
839
+ def get_running_job_ids(self) -> list[str]:
840
+ return list(self._running_jobs.keys())
841
+
842
+ async def _execute_job_wrapper(self, job_id: str) -> None:
843
+ task = asyncio.create_task(self._execute_job(job_id))
844
+ async with self._lock:
845
+ self._running_jobs[job_id] = task
846
+ try:
847
+ await task
848
+ except asyncio.CancelledError:
849
+ logger.warning("Job task cancelled: %s", job_id)
850
+ except Exception:
851
+ logger.error("Job task error: %s", job_id, exc_info=True)
852
+ finally:
853
+ async with self._lock:
854
+ self._running_jobs.pop(job_id, None)
855
+
856
+ async def _execute_job(
857
+ self,
858
+ job_id: str,
859
+ trigger_reason: TriggerReason = TriggerReason.SCHEDULED,
860
+ ) -> None:
861
+ started_at = datetime.now(UTC)
862
+ logger.info("Job execution started", extra={
863
+ "job_id": job_id, "trigger_reason": trigger_reason.value,
864
+ })
865
+
866
+ client = get_supabase_client()
867
+ if client is None:
868
+ logger.error("Supabase not available, cannot execute job %s", job_id)
869
+ return
870
+
871
+ repo = SchedulerRepository(client)
872
+ job = await repo.get_job_by_id(job_id)
873
+ if not job:
874
+ logger.error("Job not found: %s", job_id)
875
+ return
876
+
877
+ if job.get("status") not in (JobStatus.ACTIVE.value,):
878
+ logger.warning("Job %s not active, skipping", job_id)
879
+ return
880
+
881
+ # Acquire distributed lock (no-op in memory mode)
882
+ if not await self._acquire_execution_lock(job):
883
+ return
884
+
885
+ try:
886
+ # Enforce execution timeout
887
+ timeout = job.get("timeout_seconds", 30.0)
888
+ result = await asyncio.wait_for(
889
+ self._http_engine.execute(job),
890
+ timeout=timeout + 10.0,
891
+ )
892
+ except asyncio.TimeoutError:
893
+ result = HttpExecutionResult(
894
+ success=False,
895
+ error_message=f"Job execution timed out after {timeout + 10.0}s",
896
+ exception_type="ExecutionTimeout",
897
+ )
898
+ except Exception as exc:
899
+ result = HttpExecutionResult(
900
+ success=False,
901
+ error_message=str(exc),
902
+ exception_type=type(exc).__name__,
903
+ )
904
+ finally:
905
+ await self._release_execution_lock(job_id)
906
+
907
+ ended_at = datetime.now(UTC)
908
+ duration_ms = (ended_at - started_at).total_seconds() * 1000
909
+
910
+ exec_status = ExecutionStatus.SUCCESS if result.success else ExecutionStatus.FAILURE
911
+ if result.exception_type in ("TimeoutException", "ExecutionTimeout"):
912
+ exec_status = ExecutionStatus.TIMEOUT
913
+
914
+ headers_hash = hashlib.sha256(
915
+ json.dumps(job.get("headers") or {}, sort_keys=True).encode()
916
+ ).hexdigest() if job.get("headers") else ""
917
+
918
+ history_data = {
919
+ "id": str(uuid.uuid4()),
920
+ "job_id": job["id"],
921
+ "job_name": job.get("name", ""),
922
+ "started_at": started_at.isoformat(),
923
+ "ended_at": ended_at.isoformat(),
924
+ "duration_ms": round(duration_ms, 2),
925
+ "status": exec_status.value,
926
+ "http_status_code": result.status_code,
927
+ "response_size_bytes": result.response_size,
928
+ "retry_count": result.retry_count,
929
+ "trigger_reason": trigger_reason.value,
930
+ "error_message": result.error_message,
931
+ "exception_type": result.exception_type,
932
+ "request_url": job.get("url", ""),
933
+ "request_method": job.get("method", "GET"),
934
+ "response_preview": result.response_body,
935
+ }
936
+ await repo.create_history(history_data)
937
+
938
+ update_data: dict[str, Any] = {
939
+ "last_run_at": started_at.isoformat(),
940
+ "run_count": (job.get("run_count") or 0) + 1,
941
+ }
942
+ if result.success:
943
+ update_data["success_count"] = (job.get("success_count") or 0) + 1
944
+ else:
945
+ update_data["failure_count"] = (job.get("failure_count") or 0) + 1
946
+
947
+ next_run = self.get_next_run_time(job_id)
948
+ if next_run:
949
+ update_data["next_run_at"] = next_run.isoformat()
950
+
951
+ await repo.update_job(job_id, update_data)
952
+
953
+ if result.success:
954
+ logger.info("Job execution success", extra={
955
+ "job_id": job_id, "job_name": job.get("name"),
956
+ "duration_ms": round(duration_ms, 2),
957
+ "http_status": result.status_code,
958
+ "response_size": result.response_size,
959
+ "retry_count": result.retry_count,
960
+ })
961
+ else:
962
+ logger.error("Job execution failure", extra={
963
+ "job_id": job_id, "job_name": job.get("name"),
964
+ "duration_ms": round(duration_ms, 2),
965
+ "http_status": result.status_code,
966
+ "error": result.error_message,
967
+ "exception_type": result.exception_type,
968
+ "retry_count": result.retry_count,
969
+ })
970
+
971
+ # -----------------------------------------------------------------------
972
+ # Public API: Job CRUD
973
+ # -----------------------------------------------------------------------
974
+
975
+ async def create_job(self, body: dict[str, Any]) -> dict[str, Any]:
976
+ client = get_supabase_client()
977
+ if client is None:
978
+ raise RuntimeError("Supabase not available")
979
+
980
+ repo = SchedulerRepository(client)
981
+
982
+ existing = await repo.get_job_by_name(body["name"])
983
+ if existing:
984
+ raise ValueError(f"Job with name '{body['name']}' already exists")
985
+
986
+ _check_ssrf(body["url"])
987
+
988
+ job_id = str(uuid.uuid4())
989
+ now = datetime.now(UTC).isoformat()
990
+
991
+ trigger_config = dict(body.get("trigger", {}))
992
+ trigger_type = trigger_config.pop("type", "cron")
993
+
994
+ retry = body.get("retry", {})
995
+
996
+ job_data: dict[str, Any] = {
997
+ "id": job_id,
998
+ "name": body["name"],
999
+ "description": body.get("description"),
1000
+ "url": body["url"],
1001
+ "method": body.get("method", "GET"),
1002
+ "headers": body.get("headers", {}),
1003
+ "query_params": body.get("query_params", {}),
1004
+ "body": body.get("body"),
1005
+ "body_type": body.get("body_type", "json"),
1006
+ "auth_type": body.get("auth", {}).get("type", "none"),
1007
+ "auth_config": dict(body.get("auth", {})),
1008
+ "trigger_type": trigger_type,
1009
+ "trigger_config": trigger_config,
1010
+ "timezone": body.get("timezone", "UTC"),
1011
+ "timeout_seconds": body.get("timeout", 30.0),
1012
+ "max_retries": retry.get("max_retries", 3),
1013
+ "retry_delay_seconds": retry.get("retry_delay", 1.0),
1014
+ "retry_backoff": retry.get("retry_backoff", 2.0),
1015
+ "retry_max_delay_seconds": retry.get("retry_max_delay", 60.0),
1016
+ "retry_on_status": retry.get("retry_on_status", [429, 500, 502, 503, 504]),
1017
+ "retry_on_timeout": retry.get("retry_on_timeout", True),
1018
+ "retry_on_failure": retry.get("retry_on_failure", True),
1019
+ "retry_jitter": retry.get("retry_jitter", False),
1020
+ "max_instances": body.get("max_instances", 1),
1021
+ "coalesce": body.get("coalesce", True),
1022
+ "misfire_grace_time": body.get("misfire_grace_time"),
1023
+ "jitter_seconds": trigger_config.get("jitter"),
1024
+ "start_date": trigger_config.get("start_date"),
1025
+ "end_date": trigger_config.get("end_date"),
1026
+ "status": JobStatus.ACTIVE.value,
1027
+ "tags": body.get("tags", []),
1028
+ "metadata": body.get("metadata", {}),
1029
+ "created_at": now,
1030
+ "updated_at": now,
1031
+ "run_count": 0,
1032
+ "failure_count": 0,
1033
+ "success_count": 0,
1034
+ }
1035
+
1036
+ created = await repo.create_job(job_data)
1037
+ self._schedule_job(job_data)
1038
+
1039
+ next_run = self.get_next_run_time(job_id)
1040
+ if next_run:
1041
+ await repo.update_job(job_id, {"next_run_at": next_run.isoformat()})
1042
+ created["next_run_at"] = next_run.isoformat()
1043
+
1044
+ logger.info("Job created: %s (%s)", body["name"], job_id)
1045
+ return await repo.get_job_by_id(job_id) or created
1046
+
1047
+ async def update_job(self, job_id: str, body: dict[str, Any]) -> dict[str, Any]:
1048
+ client = get_supabase_client()
1049
+ if client is None:
1050
+ raise RuntimeError("Supabase not available")
1051
+
1052
+ repo = SchedulerRepository(client)
1053
+ job = await repo.get_job_by_id(job_id)
1054
+ if not job:
1055
+ raise KeyError(f"Job '{job_id}' not found")
1056
+ if job.get("status") == JobStatus.DELETED.value:
1057
+ raise ValueError("Cannot update a deleted job")
1058
+
1059
+ update_data: dict[str, Any] = {}
1060
+
1061
+ field_map = {
1062
+ "description": "description",
1063
+ "url": "url",
1064
+ "method": "method",
1065
+ "headers": "headers",
1066
+ "query_params": "query_params",
1067
+ "body": "body",
1068
+ "body_type": "body_type",
1069
+ "timezone": "timezone",
1070
+ "timeout": "timeout_seconds",
1071
+ "max_instances": "max_instances",
1072
+ "coalesce": "coalesce",
1073
+ "misfire_grace_time": "misfire_grace_time",
1074
+ "tags": "tags",
1075
+ "metadata": "metadata",
1076
+ }
1077
+
1078
+ for req_key, db_key in field_map.items():
1079
+ if req_key in body and body[req_key] is not None:
1080
+ update_data[db_key] = body[req_key]
1081
+
1082
+ if "url" in body and body["url"] is not None:
1083
+ _check_ssrf(body["url"])
1084
+
1085
+ if "auth" in body and body["auth"] is not None:
1086
+ update_data["auth_type"] = body["auth"].get("type", "none")
1087
+ update_data["auth_config"] = dict(body["auth"])
1088
+
1089
+ if "retry" in body and body["retry"] is not None:
1090
+ r = body["retry"]
1091
+ update_data["max_retries"] = r.get("max_retries", 3)
1092
+ update_data["retry_delay_seconds"] = r.get("retry_delay", 1.0)
1093
+ update_data["retry_backoff"] = r.get("retry_backoff", 2.0)
1094
+ update_data["retry_max_delay_seconds"] = r.get("retry_max_delay", 60.0)
1095
+ update_data["retry_on_status"] = r.get("retry_on_status", [429, 500, 502, 503, 504])
1096
+ update_data["retry_on_timeout"] = r.get("retry_on_timeout", True)
1097
+ update_data["retry_on_failure"] = r.get("retry_on_failure", True)
1098
+ update_data["retry_jitter"] = r.get("retry_jitter", False)
1099
+
1100
+ if "trigger" in body and body["trigger"] is not None:
1101
+ tc = dict(body["trigger"])
1102
+ update_data["trigger_type"] = tc.pop("type", job.get("trigger_type", "cron"))
1103
+ update_data["trigger_config"] = tc
1104
+ update_data["jitter_seconds"] = tc.get("jitter")
1105
+ update_data["start_date"] = tc.get("start_date")
1106
+ update_data["end_date"] = tc.get("end_date")
1107
+
1108
+ await repo.update_job(job_id, update_data)
1109
+ updated = await repo.get_job_by_id(job_id)
1110
+ if updated is None:
1111
+ raise KeyError(f"Job '{job_id}' not found after update")
1112
+
1113
+ self._reschedule_job(updated)
1114
+
1115
+ logger.info("Job updated: %s", job_id)
1116
+ return updated
1117
+
1118
+ async def delete_job(self, job_id: str) -> None:
1119
+ client = get_supabase_client()
1120
+ if client is None:
1121
+ raise RuntimeError("Supabase not available")
1122
+
1123
+ repo = SchedulerRepository(client)
1124
+ job = await repo.get_job_by_id(job_id)
1125
+ if not job:
1126
+ raise KeyError(f"Job '{job_id}' not found")
1127
+
1128
+ self._remove_aps_job(job_id)
1129
+ await repo.update_job(job_id, {"status": JobStatus.DELETED.value})
1130
+ logger.info("Job deleted: %s", job_id)
1131
+
1132
+ async def hard_delete_job(self, job_id: str) -> None:
1133
+ client = get_supabase_client()
1134
+ if client is None:
1135
+ raise RuntimeError("Supabase not available")
1136
+
1137
+ repo = SchedulerRepository(client)
1138
+ job = await repo.get_job_by_id(job_id)
1139
+ if not job:
1140
+ raise KeyError(f"Job '{job_id}' not found")
1141
+
1142
+ self._remove_aps_job(job_id)
1143
+ await repo.delete_job(job_id)
1144
+ logger.info("Job hard deleted: %s", job_id)
1145
+
1146
+ async def pause_job(self, job_id: str) -> dict[str, Any]:
1147
+ client = get_supabase_client()
1148
+ if client is None:
1149
+ raise RuntimeError("Supabase not available")
1150
+
1151
+ repo = SchedulerRepository(client)
1152
+ job = await repo.get_job_by_id(job_id)
1153
+ if not job:
1154
+ raise KeyError(f"Job '{job_id}' not found")
1155
+ if job.get("status") != JobStatus.ACTIVE.value:
1156
+ raise ValueError(f"Job is not active (current status: {job.get('status')})")
1157
+
1158
+ self.pause_aps_job(job_id)
1159
+ await repo.update_job(job_id, {"status": JobStatus.PAUSED.value})
1160
+ updated = await repo.get_job_by_id(job_id)
1161
+ logger.info("Job paused: %s", job_id)
1162
+ return updated or job
1163
+
1164
+ async def resume_job(self, job_id: str) -> dict[str, Any]:
1165
+ client = get_supabase_client()
1166
+ if client is None:
1167
+ raise RuntimeError("Supabase not available")
1168
+
1169
+ repo = SchedulerRepository(client)
1170
+ job = await repo.get_job_by_id(job_id)
1171
+ if not job:
1172
+ raise KeyError(f"Job '{job_id}' not found")
1173
+ if job.get("status") != JobStatus.PAUSED.value:
1174
+ raise ValueError(f"Job is not paused (current status: {job.get('status')})")
1175
+
1176
+ self.resume_aps_job(job_id)
1177
+ await repo.update_job(job_id, {"status": JobStatus.ACTIVE.value})
1178
+ updated = await repo.get_job_by_id(job_id)
1179
+ logger.info("Job resumed: %s", job_id)
1180
+ return updated or job
1181
+
1182
+ async def run_job_now(self, job_id: str) -> None:
1183
+ client = get_supabase_client()
1184
+ if client is None:
1185
+ raise RuntimeError("Supabase not available")
1186
+
1187
+ repo = SchedulerRepository(client)
1188
+ job = await repo.get_job_by_id(job_id)
1189
+ if not job:
1190
+ raise KeyError(f"Job '{job_id}' not found")
1191
+ if job.get("status") == JobStatus.DELETED.value:
1192
+ raise ValueError("Cannot run a deleted job")
1193
+
1194
+ asyncio.create_task(self._execute_job(job_id, TriggerReason.MANUAL))
1195
+ logger.info("Job triggered manually: %s", job_id)
1196
+
1197
+ async def list_jobs(
1198
+ self,
1199
+ status: str | None = None,
1200
+ tags: list[str] | None = None,
1201
+ page: int = 1,
1202
+ page_size: int = 20,
1203
+ ) -> tuple[list[dict[str, Any]], int]:
1204
+ client = get_supabase_client()
1205
+ if client is None:
1206
+ return [], 0
1207
+ repo = SchedulerRepository(client)
1208
+ return await repo.list_jobs(status=status, tags=tags, page=page, page_size=page_size)
1209
+
1210
+ async def get_job(self, job_id: str) -> dict[str, Any] | None:
1211
+ client = get_supabase_client()
1212
+ if client is None:
1213
+ return None
1214
+ repo = SchedulerRepository(client)
1215
+ return await repo.get_job_by_id(job_id)
1216
+
1217
+ async def get_job_history(
1218
+ self,
1219
+ job_id: str,
1220
+ page: int = 1,
1221
+ page_size: int = 20,
1222
+ ) -> tuple[list[dict[str, Any]], int]:
1223
+ client = get_supabase_client()
1224
+ if client is None:
1225
+ return [], 0
1226
+ repo = SchedulerRepository(client)
1227
+ return await repo.get_history(job_id=job_id, page=page, page_size=page_size)
1228
+
1229
+ async def get_execution_history(
1230
+ self,
1231
+ page: int = 1,
1232
+ page_size: int = 50,
1233
+ status: str | None = None,
1234
+ ) -> tuple[list[dict[str, Any]], int]:
1235
+ client = get_supabase_client()
1236
+ if client is None:
1237
+ return [], 0
1238
+ repo = SchedulerRepository(client)
1239
+ return await repo.get_history_all(page=page, page_size=page_size, status_filter=status)
1240
+
1241
+ async def get_metrics(self) -> dict[str, Any]:
1242
+ client = get_supabase_client()
1243
+ if client is None:
1244
+ return {"error": "Supabase not available"}
1245
+
1246
+ repo = SchedulerRepository(client)
1247
+ total_jobs = await repo.count_jobs()
1248
+ active_jobs = await repo.count_jobs(status=JobStatus.ACTIVE.value)
1249
+ paused_jobs = await repo.count_jobs(status=JobStatus.PAUSED.value)
1250
+ total_executions = await repo.count_executions()
1251
+ successful_executions = await repo.count_executions(status=ExecutionStatus.SUCCESS.value)
1252
+
1253
+ scheduler_status = self.get_scheduler_status()
1254
+
1255
+ return {
1256
+ "jobs": {
1257
+ "total": total_jobs,
1258
+ "active": active_jobs,
1259
+ "paused": paused_jobs,
1260
+ },
1261
+ "executions": {
1262
+ "total": total_executions,
1263
+ "success": successful_executions,
1264
+ "failure": total_executions - successful_executions,
1265
+ "success_rate": round(successful_executions / total_executions * 100, 2) if total_executions else 0,
1266
+ },
1267
+ "scheduler": scheduler_status,
1268
+ }
1269
+
1270
+
1271
+ # ---------------------------------------------------------------------------
1272
+ # Validation helpers
1273
+ # ---------------------------------------------------------------------------
1274
+
1275
+ def validate_cron_expression(expression: str) -> None:
1276
+ try:
1277
+ croniter(expression)
1278
+ except (ValueError, KeyError) as exc:
1279
+ raise ValueError(f"Invalid cron expression '{expression}': {exc}") from exc
1280
+
1281
+
1282
+ def validate_timezone(tz: str) -> None:
1283
+ try:
1284
+ import pytz
1285
+ pytz.timezone(tz)
1286
+ except Exception as exc:
1287
+ raise ValueError(f"Unknown timezone: '{tz}'") from exc
1288
+
1289
+
1290
+ def validate_url(url: str) -> str:
1291
+ if not url.startswith(("http://", "https://")):
1292
+ raise ValueError("Only http/https URLs are supported")
1293
+ return url
1294
+
1295
+
1296
+ def job_to_response(job: dict[str, Any]) -> dict[str, Any]:
1297
+ return {
1298
+ "id": job.get("id"),
1299
+ "name": job.get("name"),
1300
+ "description": job.get("description"),
1301
+ "url": job.get("url"),
1302
+ "method": job.get("method", "GET"),
1303
+ "headers": job.get("headers", {}),
1304
+ "query_params": job.get("query_params", {}),
1305
+ "body": job.get("body"),
1306
+ "body_type": job.get("body_type", "json"),
1307
+ "auth_type": job.get("auth_type", "none"),
1308
+ "trigger_type": job.get("trigger_type"),
1309
+ "trigger_config": job.get("trigger_config", {}),
1310
+ "timezone": job.get("timezone", "UTC"),
1311
+ "timeout": job.get("timeout_seconds", 30.0),
1312
+ "max_retries": job.get("max_retries", 3),
1313
+ "retry_delay": job.get("retry_delay_seconds", 1.0),
1314
+ "retry_backoff": job.get("retry_backoff", 2.0),
1315
+ "retry_max_delay": job.get("retry_max_delay_seconds", 60.0),
1316
+ "retry_on_status": job.get("retry_on_status", [429, 500, 502, 503, 504]),
1317
+ "retry_on_timeout": job.get("retry_on_timeout", True),
1318
+ "retry_on_failure": job.get("retry_on_failure", True),
1319
+ "retry_jitter": job.get("retry_jitter", False),
1320
+ "max_instances": job.get("max_instances", 1),
1321
+ "coalesce": job.get("coalesce", True),
1322
+ "misfire_grace_time": job.get("misfire_grace_time"),
1323
+ "jitter": job.get("jitter_seconds"),
1324
+ "start_date": job.get("start_date"),
1325
+ "end_date": job.get("end_date"),
1326
+ "status": job.get("status", "active"),
1327
+ "tags": job.get("tags", []),
1328
+ "metadata": job.get("metadata", {}),
1329
+ "created_at": job.get("created_at"),
1330
+ "updated_at": job.get("updated_at"),
1331
+ "last_run_at": job.get("last_run_at"),
1332
+ "next_run_at": job.get("next_run_at"),
1333
+ "run_count": job.get("run_count", 0),
1334
+ "failure_count": job.get("failure_count", 0),
1335
+ "success_count": job.get("success_count", 0),
1336
+ }
1337
+
1338
+
1339
+ def history_to_response(h: dict[str, Any]) -> dict[str, Any]:
1340
+ return {
1341
+ "id": h.get("id"),
1342
+ "job_id": h.get("job_id"),
1343
+ "job_name": h.get("job_name"),
1344
+ "started_at": h.get("started_at"),
1345
+ "ended_at": h.get("ended_at"),
1346
+ "duration_ms": h.get("duration_ms"),
1347
+ "status": h.get("status"),
1348
+ "http_status_code": h.get("http_status_code"),
1349
+ "response_size_bytes": h.get("response_size_bytes"),
1350
+ "retry_count": h.get("retry_count", 0),
1351
+ "trigger_reason": h.get("trigger_reason", "scheduled"),
1352
+ "error_message": h.get("error_message"),
1353
+ "exception_type": h.get("exception_type"),
1354
+ "request_url": h.get("request_url"),
1355
+ "request_method": h.get("request_method"),
1356
+ "response_preview": h.get("response_preview"),
1357
+ "created_at": h.get("created_at"),
1358
+ }
pyproject.toml CHANGED
@@ -33,6 +33,10 @@ dependencies = [
33
  "phonenumbers>=8.13.0",
34
  "jsonschema>=4.21.0",
35
  "supabase>=2.0.0",
 
 
 
 
36
  ]
37
 
38
  [project.optional-dependencies]
 
33
  "phonenumbers>=8.13.0",
34
  "jsonschema>=4.21.0",
35
  "supabase>=2.0.0",
36
+ "apscheduler>=3.10.4",
37
+ "croniter>=2.0.0",
38
+ "pytz>=2024.1",
39
+ "redis>=5.0.0",
40
  ]
41
 
42
  [project.optional-dependencies]
requirements.txt CHANGED
@@ -67,3 +67,8 @@ jsonpath-ng>=1.7.0
67
 
68
  # QR code generation (testing)
69
  qrcode[pil]>=8.0
 
 
 
 
 
 
67
 
68
  # QR code generation (testing)
69
  qrcode[pil]>=8.0
70
+
71
+ # Scheduler
72
+ apscheduler>=3.10.4
73
+ croniter>=2.0.0
74
+ pytz>=2024.1