Spaces:
Running
Running
Commit ·
947ea10
1
Parent(s): 5d6260a
feat: cron job api added
Browse files- .gitignore +3 -1
- app/api/deps.py +6 -0
- app/api/server.py +7 -0
- app/api/v1/router.py +2 -0
- app/api/v1/scheduler.py +326 -0
- app/config.py +22 -0
- app/services/scheduler_service.py +1358 -0
- pyproject.toml +4 -0
- requirements.txt +5 -0
.gitignore
CHANGED
|
@@ -134,4 +134,6 @@ tests
|
|
| 134 |
deploy_hf.py
|
| 135 |
.mimocode
|
| 136 |
|
| 137 |
-
ddl
|
|
|
|
|
|
|
|
|
| 134 |
deploy_hf.py
|
| 135 |
.mimocode
|
| 136 |
|
| 137 |
+
ddl
|
| 138 |
+
API_DESCRIPTION.md
|
| 139 |
+
ENTERPRISE-API-ROADMAP*.md
|
app/api/deps.py
CHANGED
|
@@ -13,6 +13,7 @@ from app.services.extraction_service import ExtractionService
|
|
| 13 |
from app.services.ocr_service import OCRService
|
| 14 |
from app.services.sql_validator_service import SqlValidatorService
|
| 15 |
from app.services.text_cleaner_service import TextCleanerService
|
|
|
|
| 16 |
from app.services.vector_store_service import VectorStoreService
|
| 17 |
|
| 18 |
|
|
@@ -54,6 +55,11 @@ def get_vector_store_service() -> VectorStoreService:
|
|
| 54 |
return _vector_store_service
|
| 55 |
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
def require_auth(token: str = Depends(require_api_key)) -> str:
|
| 58 |
return token
|
| 59 |
|
|
|
|
| 13 |
from app.services.ocr_service import OCRService
|
| 14 |
from app.services.sql_validator_service import SqlValidatorService
|
| 15 |
from app.services.text_cleaner_service import TextCleanerService
|
| 16 |
+
from app.services.scheduler_service import SchedulerService
|
| 17 |
from app.services.vector_store_service import VectorStoreService
|
| 18 |
|
| 19 |
|
|
|
|
| 55 |
return _vector_store_service
|
| 56 |
|
| 57 |
|
| 58 |
+
def get_scheduler_service() -> SchedulerService:
|
| 59 |
+
from app.api.server import _scheduler_service
|
| 60 |
+
return _scheduler_service
|
| 61 |
+
|
| 62 |
+
|
| 63 |
def require_auth(token: str = Depends(require_api_key)) -> str:
|
| 64 |
return token
|
| 65 |
|
app/api/server.py
CHANGED
|
@@ -16,6 +16,7 @@ from app.core.logger import get_logger
|
|
| 16 |
from app.core.redis_client import close_redis, create_redis_client
|
| 17 |
from app.core.scripts import load_scripts
|
| 18 |
from app.services.embeddings_service import EmbeddingService
|
|
|
|
| 19 |
from app.services.vector_store_service import VectorStoreService
|
| 20 |
|
| 21 |
_logger = get_logger(__name__)
|
|
@@ -23,6 +24,7 @@ _settings = get_settings()
|
|
| 23 |
|
| 24 |
_embedding_service: EmbeddingService = EmbeddingService()
|
| 25 |
_vector_store_service: VectorStoreService = VectorStoreService(_embedding_service)
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
async def _self_ping():
|
|
@@ -78,8 +80,13 @@ async def lifespan(app: FastAPI):
|
|
| 78 |
_logger.warning("Redis not configured, running in degraded mode")
|
| 79 |
|
| 80 |
asyncio.create_task(_self_ping())
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
yield
|
| 82 |
_logger.info("Shutting down...")
|
|
|
|
| 83 |
await close_redis(redis)
|
| 84 |
await _vector_store_service.close_all()
|
| 85 |
await pool_manager.close_all()
|
|
|
|
| 16 |
from app.core.redis_client import close_redis, create_redis_client
|
| 17 |
from app.core.scripts import load_scripts
|
| 18 |
from app.services.embeddings_service import EmbeddingService
|
| 19 |
+
from app.services.scheduler_service import SchedulerService
|
| 20 |
from app.services.vector_store_service import VectorStoreService
|
| 21 |
|
| 22 |
_logger = get_logger(__name__)
|
|
|
|
| 24 |
|
| 25 |
_embedding_service: EmbeddingService = EmbeddingService()
|
| 26 |
_vector_store_service: VectorStoreService = VectorStoreService(_embedding_service)
|
| 27 |
+
_scheduler_service: SchedulerService = SchedulerService()
|
| 28 |
|
| 29 |
|
| 30 |
async def _self_ping():
|
|
|
|
| 80 |
_logger.warning("Redis not configured, running in degraded mode")
|
| 81 |
|
| 82 |
asyncio.create_task(_self_ping())
|
| 83 |
+
|
| 84 |
+
await _scheduler_service.start()
|
| 85 |
+
_logger.info("Scheduler service started")
|
| 86 |
+
|
| 87 |
yield
|
| 88 |
_logger.info("Shutting down...")
|
| 89 |
+
await _scheduler_service.shutdown()
|
| 90 |
await close_redis(redis)
|
| 91 |
await _vector_store_service.close_all()
|
| 92 |
await pool_manager.close_all()
|
app/api/v1/router.py
CHANGED
|
@@ -16,6 +16,7 @@ from app.api.v1 import (
|
|
| 16 |
qr_decoder,
|
| 17 |
qr_generator,
|
| 18 |
reconcile,
|
|
|
|
| 19 |
scraper,
|
| 20 |
semantic_router,
|
| 21 |
sql_validator,
|
|
@@ -39,6 +40,7 @@ api_v1_router.include_router(embeddings.router, tags=["Embeddings"])
|
|
| 39 |
api_v1_router.include_router(code_executor.router, tags=["Code Executor"])
|
| 40 |
api_v1_router.include_router(verify_router, prefix="/verify", tags=["Verify"])
|
| 41 |
api_v1_router.include_router(reconcile.router, tags=["Reconcile"])
|
|
|
|
| 42 |
api_v1_router.include_router(scraper.router, tags=["Web Scraping"])
|
| 43 |
api_v1_router.include_router(web_search.router, tags=["Web Search"])
|
| 44 |
api_v1_router.include_router(sql_validator.router, tags=["SQL Validator"])
|
|
|
|
| 16 |
qr_decoder,
|
| 17 |
qr_generator,
|
| 18 |
reconcile,
|
| 19 |
+
scheduler,
|
| 20 |
scraper,
|
| 21 |
semantic_router,
|
| 22 |
sql_validator,
|
|
|
|
| 40 |
api_v1_router.include_router(code_executor.router, tags=["Code Executor"])
|
| 41 |
api_v1_router.include_router(verify_router, prefix="/verify", tags=["Verify"])
|
| 42 |
api_v1_router.include_router(reconcile.router, tags=["Reconcile"])
|
| 43 |
+
api_v1_router.include_router(scheduler.router, tags=["Scheduler"])
|
| 44 |
api_v1_router.include_router(scraper.router, tags=["Web Scraping"])
|
| 45 |
api_v1_router.include_router(web_search.router, tags=["Web Search"])
|
| 46 |
api_v1_router.include_router(sql_validator.router, tags=["SQL Validator"])
|
app/api/v1/scheduler.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 6 |
+
|
| 7 |
+
from app.api.deps import get_scheduler_service, require_auth
|
| 8 |
+
from app.core.logger import get_logger
|
| 9 |
+
from app.services.scheduler_service import (
|
| 10 |
+
SchedulerService,
|
| 11 |
+
job_to_response,
|
| 12 |
+
history_to_response,
|
| 13 |
+
validate_cron_expression,
|
| 14 |
+
validate_timezone,
|
| 15 |
+
validate_url,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
router = APIRouter()
|
| 19 |
+
logger = get_logger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _paginate(items: list[Any], total: int, page: int, page_size: int) -> dict[str, Any]:
|
| 23 |
+
return {
|
| 24 |
+
"items": items,
|
| 25 |
+
"total": total,
|
| 26 |
+
"page": page,
|
| 27 |
+
"page_size": page_size,
|
| 28 |
+
"has_next": (page * page_size) < total,
|
| 29 |
+
"has_prev": page > 1,
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ---------------------------------------------------------------------------
|
| 34 |
+
# Scheduler Monitoring
|
| 35 |
+
# ---------------------------------------------------------------------------
|
| 36 |
+
|
| 37 |
+
@router.get("/scheduler/status", summary="Get scheduler status")
|
| 38 |
+
async def get_scheduler_status(
|
| 39 |
+
token: str = Depends(require_auth),
|
| 40 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 41 |
+
):
|
| 42 |
+
status = scheduler_service.get_scheduler_status()
|
| 43 |
+
status["running_job_ids"] = scheduler_service.get_running_job_ids()
|
| 44 |
+
return {"success": True, "data": status}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@router.get("/scheduler/metrics", summary="Get scheduler metrics")
|
| 48 |
+
async def get_scheduler_metrics(
|
| 49 |
+
token: str = Depends(require_auth),
|
| 50 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 51 |
+
):
|
| 52 |
+
metrics = await scheduler_service.get_metrics()
|
| 53 |
+
return {"success": True, "data": metrics}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@router.get("/scheduler/health", summary="Get scheduler health (Redis + instance status)")
|
| 57 |
+
async def get_scheduler_health(
|
| 58 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 59 |
+
):
|
| 60 |
+
health = await scheduler_service.get_health()
|
| 61 |
+
return {"success": True, "data": health}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# ---------------------------------------------------------------------------
|
| 65 |
+
# Job CRUD
|
| 66 |
+
# ---------------------------------------------------------------------------
|
| 67 |
+
|
| 68 |
+
@router.post("/scheduler/jobs", summary="Create a new scheduled job", status_code=201)
|
| 69 |
+
async def create_job(
|
| 70 |
+
body: dict[str, Any],
|
| 71 |
+
token: str = Depends(require_auth),
|
| 72 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 73 |
+
):
|
| 74 |
+
errors = _validate_job_create(body)
|
| 75 |
+
if errors:
|
| 76 |
+
raise HTTPException(status_code=422, detail={"success": False, "errors": errors})
|
| 77 |
+
|
| 78 |
+
try:
|
| 79 |
+
job = await scheduler_service.create_job(body)
|
| 80 |
+
return {"success": True, "data": job_to_response(job)}
|
| 81 |
+
except ValueError as exc:
|
| 82 |
+
raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@router.get("/scheduler/jobs", summary="List all scheduled jobs")
|
| 86 |
+
async def list_jobs(
|
| 87 |
+
status: str | None = Query(None, description="Filter by status (active, paused, completed, failed, deleted)"),
|
| 88 |
+
tags: str | None = Query(None, description="Comma-separated tags to filter by"),
|
| 89 |
+
page: int = Query(1, ge=1, description="Page number"),
|
| 90 |
+
page_size: int = Query(20, ge=1, le=100, description="Items per page"),
|
| 91 |
+
token: str = Depends(require_auth),
|
| 92 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 93 |
+
):
|
| 94 |
+
tag_list = [t.strip() for t in tags.split(",") if t.strip()] if tags else None
|
| 95 |
+
jobs, total = await scheduler_service.list_jobs(
|
| 96 |
+
status=status, tags=tag_list, page=page, page_size=page_size,
|
| 97 |
+
)
|
| 98 |
+
return {
|
| 99 |
+
"success": True,
|
| 100 |
+
"data": _paginate(
|
| 101 |
+
items=[job_to_response(j) for j in jobs],
|
| 102 |
+
total=total, page=page, page_size=page_size,
|
| 103 |
+
),
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@router.get("/scheduler/jobs/{job_id}", summary="Get a scheduled job by ID")
|
| 108 |
+
async def get_job(
|
| 109 |
+
job_id: str,
|
| 110 |
+
token: str = Depends(require_auth),
|
| 111 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 112 |
+
):
|
| 113 |
+
job = await scheduler_service.get_job(job_id)
|
| 114 |
+
if not job or job.get("status") == "deleted":
|
| 115 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": "Job not found"})
|
| 116 |
+
return {"success": True, "data": job_to_response(job)}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
@router.put("/scheduler/jobs/{job_id}", summary="Update a scheduled job")
|
| 120 |
+
async def update_job(
|
| 121 |
+
job_id: str,
|
| 122 |
+
body: dict[str, Any],
|
| 123 |
+
token: str = Depends(require_auth),
|
| 124 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 125 |
+
):
|
| 126 |
+
try:
|
| 127 |
+
job = await scheduler_service.update_job(job_id, body)
|
| 128 |
+
return {"success": True, "data": job_to_response(job)}
|
| 129 |
+
except KeyError as exc:
|
| 130 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 131 |
+
except ValueError as exc:
|
| 132 |
+
raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
@router.delete("/scheduler/jobs/{job_id}", summary="Delete a scheduled job (soft delete)")
|
| 136 |
+
async def delete_job(
|
| 137 |
+
job_id: str,
|
| 138 |
+
token: str = Depends(require_auth),
|
| 139 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 140 |
+
):
|
| 141 |
+
try:
|
| 142 |
+
await scheduler_service.delete_job(job_id)
|
| 143 |
+
return {"success": True, "message": "Job deleted successfully"}
|
| 144 |
+
except KeyError as exc:
|
| 145 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
@router.delete("/scheduler/jobs/{job_id}/hard", summary="Permanently delete a scheduled job")
|
| 149 |
+
async def hard_delete_job(
|
| 150 |
+
job_id: str,
|
| 151 |
+
token: str = Depends(require_auth),
|
| 152 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 153 |
+
):
|
| 154 |
+
try:
|
| 155 |
+
await scheduler_service.hard_delete_job(job_id)
|
| 156 |
+
return {"success": True, "message": "Job permanently deleted"}
|
| 157 |
+
except KeyError as exc:
|
| 158 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# ---------------------------------------------------------------------------
|
| 162 |
+
# Job Control
|
| 163 |
+
# ---------------------------------------------------------------------------
|
| 164 |
+
|
| 165 |
+
@router.post("/scheduler/jobs/{job_id}/pause", summary="Pause a scheduled job")
|
| 166 |
+
async def pause_job(
|
| 167 |
+
job_id: str,
|
| 168 |
+
token: str = Depends(require_auth),
|
| 169 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 170 |
+
):
|
| 171 |
+
try:
|
| 172 |
+
job = await scheduler_service.pause_job(job_id)
|
| 173 |
+
return {"success": True, "data": job_to_response(job)}
|
| 174 |
+
except KeyError as exc:
|
| 175 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 176 |
+
except ValueError as exc:
|
| 177 |
+
raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
@router.post("/scheduler/jobs/{job_id}/resume", summary="Resume a paused job")
|
| 181 |
+
async def resume_job(
|
| 182 |
+
job_id: str,
|
| 183 |
+
token: str = Depends(require_auth),
|
| 184 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 185 |
+
):
|
| 186 |
+
try:
|
| 187 |
+
job = await scheduler_service.resume_job(job_id)
|
| 188 |
+
return {"success": True, "data": job_to_response(job)}
|
| 189 |
+
except KeyError as exc:
|
| 190 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 191 |
+
except ValueError as exc:
|
| 192 |
+
raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
@router.post("/scheduler/jobs/{job_id}/run", summary="Trigger a job execution immediately")
|
| 196 |
+
async def run_job_now(
|
| 197 |
+
job_id: str,
|
| 198 |
+
token: str = Depends(require_auth),
|
| 199 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 200 |
+
):
|
| 201 |
+
try:
|
| 202 |
+
await scheduler_service.run_job_now(job_id)
|
| 203 |
+
return {"success": True, "message": "Job execution triggered"}
|
| 204 |
+
except KeyError as exc:
|
| 205 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": str(exc)}) from exc
|
| 206 |
+
except ValueError as exc:
|
| 207 |
+
raise HTTPException(status_code=400, detail={"success": False, "error": str(exc)}) from exc
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
# ---------------------------------------------------------------------------
|
| 211 |
+
# Execution History
|
| 212 |
+
# ---------------------------------------------------------------------------
|
| 213 |
+
|
| 214 |
+
@router.get("/scheduler/jobs/{job_id}/history", summary="Get execution history for a job")
|
| 215 |
+
async def get_job_history(
|
| 216 |
+
job_id: str,
|
| 217 |
+
page: int = Query(1, ge=1, description="Page number"),
|
| 218 |
+
page_size: int = Query(20, ge=1, le=100, description="Items per page"),
|
| 219 |
+
token: str = Depends(require_auth),
|
| 220 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 221 |
+
):
|
| 222 |
+
job = await scheduler_service.get_job(job_id)
|
| 223 |
+
if not job:
|
| 224 |
+
raise HTTPException(status_code=404, detail={"success": False, "error": "Job not found"})
|
| 225 |
+
|
| 226 |
+
history, total = await scheduler_service.get_job_history(
|
| 227 |
+
job_id=job_id, page=page, page_size=page_size,
|
| 228 |
+
)
|
| 229 |
+
return {
|
| 230 |
+
"success": True,
|
| 231 |
+
"data": _paginate(
|
| 232 |
+
items=[history_to_response(h) for h in history],
|
| 233 |
+
total=total, page=page, page_size=page_size,
|
| 234 |
+
),
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
@router.get("/scheduler/history", summary="Get global execution history")
|
| 239 |
+
async def get_execution_history(
|
| 240 |
+
page: int = Query(1, ge=1, description="Page number"),
|
| 241 |
+
page_size: int = Query(50, ge=1, le=200, description="Items per page"),
|
| 242 |
+
status: str | None = Query(None, description="Filter by execution status"),
|
| 243 |
+
token: str = Depends(require_auth),
|
| 244 |
+
scheduler_service: SchedulerService = Depends(get_scheduler_service),
|
| 245 |
+
):
|
| 246 |
+
history, total = await scheduler_service.get_execution_history(
|
| 247 |
+
page=page, page_size=page_size, status=status,
|
| 248 |
+
)
|
| 249 |
+
return {
|
| 250 |
+
"success": True,
|
| 251 |
+
"data": _paginate(
|
| 252 |
+
items=[history_to_response(h) for h in history],
|
| 253 |
+
total=total, page=page, page_size=page_size,
|
| 254 |
+
),
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
# ---------------------------------------------------------------------------
|
| 259 |
+
# Validation
|
| 260 |
+
# ---------------------------------------------------------------------------
|
| 261 |
+
|
| 262 |
+
def _validate_job_create(body: dict[str, Any]) -> list[str]:
|
| 263 |
+
errors: list[str] = []
|
| 264 |
+
|
| 265 |
+
if not body.get("name"):
|
| 266 |
+
errors.append("name is required")
|
| 267 |
+
elif not isinstance(body["name"], str) or len(body["name"]) > 255:
|
| 268 |
+
errors.append("name must be a string between 1 and 255 characters")
|
| 269 |
+
|
| 270 |
+
if not body.get("url"):
|
| 271 |
+
errors.append("url is required")
|
| 272 |
+
elif not isinstance(body["url"], str):
|
| 273 |
+
errors.append("url must be a string")
|
| 274 |
+
else:
|
| 275 |
+
try:
|
| 276 |
+
validate_url(body["url"])
|
| 277 |
+
except ValueError as exc:
|
| 278 |
+
errors.append(str(exc))
|
| 279 |
+
|
| 280 |
+
trigger = body.get("trigger")
|
| 281 |
+
if not trigger:
|
| 282 |
+
errors.append("trigger is required")
|
| 283 |
+
elif not isinstance(trigger, dict):
|
| 284 |
+
errors.append("trigger must be an object")
|
| 285 |
+
else:
|
| 286 |
+
trigger_type = trigger.get("type")
|
| 287 |
+
if not trigger_type:
|
| 288 |
+
errors.append("trigger.type is required")
|
| 289 |
+
elif trigger_type not in ("cron", "interval", "date"):
|
| 290 |
+
errors.append("trigger.type must be one of: cron, interval, date")
|
| 291 |
+
elif trigger_type == "cron":
|
| 292 |
+
if not trigger.get("cron_expression"):
|
| 293 |
+
errors.append("trigger.cron_expression is required for cron trigger")
|
| 294 |
+
else:
|
| 295 |
+
try:
|
| 296 |
+
validate_cron_expression(trigger["cron_expression"])
|
| 297 |
+
except ValueError as exc:
|
| 298 |
+
errors.append(str(exc))
|
| 299 |
+
|
| 300 |
+
if body.get("timezone"):
|
| 301 |
+
try:
|
| 302 |
+
validate_timezone(body["timezone"])
|
| 303 |
+
except ValueError as exc:
|
| 304 |
+
errors.append(str(exc))
|
| 305 |
+
|
| 306 |
+
timeout = body.get("timeout")
|
| 307 |
+
if timeout is not None:
|
| 308 |
+
if not isinstance(timeout, (int, float)):
|
| 309 |
+
errors.append("timeout must be a number")
|
| 310 |
+
elif timeout < 1 or timeout > 300:
|
| 311 |
+
errors.append("timeout must be between 1 and 300")
|
| 312 |
+
|
| 313 |
+
method = body.get("method")
|
| 314 |
+
if method and method not in ("GET", "POST", "PUT", "PATCH", "DELETE"):
|
| 315 |
+
errors.append("method must be one of: GET, POST, PUT, PATCH, DELETE")
|
| 316 |
+
|
| 317 |
+
retry = body.get("retry")
|
| 318 |
+
if retry is not None:
|
| 319 |
+
if not isinstance(retry, dict):
|
| 320 |
+
errors.append("retry must be an object")
|
| 321 |
+
else:
|
| 322 |
+
max_retries = retry.get("max_retries")
|
| 323 |
+
if max_retries is not None and (not isinstance(max_retries, int) or max_retries < 0 or max_retries > 10):
|
| 324 |
+
errors.append("retry.max_retries must be between 0 and 10")
|
| 325 |
+
|
| 326 |
+
return errors
|
app/config.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
| 3 |
from functools import lru_cache
|
| 4 |
from typing import Optional
|
| 5 |
|
|
|
|
| 6 |
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 7 |
|
| 8 |
|
|
@@ -83,6 +84,27 @@ class Settings(BaseSettings):
|
|
| 83 |
lockout_minutes: int = 15
|
| 84 |
admin_password: str = ""
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
@property
|
| 87 |
def max_upload_mb(self) -> int:
|
| 88 |
return self.max_upload_bytes // (1024 * 1024)
|
|
|
|
| 3 |
from functools import lru_cache
|
| 4 |
from typing import Optional
|
| 5 |
|
| 6 |
+
from pydantic import Field
|
| 7 |
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 8 |
|
| 9 |
|
|
|
|
| 84 |
lockout_minutes: int = 15
|
| 85 |
admin_password: str = ""
|
| 86 |
|
| 87 |
+
# Redis connection
|
| 88 |
+
redis_host: str = Field(default="localhost", alias="REDIS_HOST")
|
| 89 |
+
redis_port: int = Field(default=6379, alias="REDIS_PORT")
|
| 90 |
+
redis_db: int = Field(default=0, alias="REDIS_DB")
|
| 91 |
+
redis_password: str | None = Field(default=None, alias="REDIS_PASSWORD")
|
| 92 |
+
redis_ssl: bool = Field(default=False, alias="REDIS_SSL")
|
| 93 |
+
redis_socket_timeout: int = Field(default=30, alias="REDIS_SOCKET_TIMEOUT")
|
| 94 |
+
redis_socket_connect_timeout: int = Field(default=30, alias="REDIS_SOCKET_CONNECT_TIMEOUT")
|
| 95 |
+
redis_retry_on_timeout: bool = Field(default=True, alias="REDIS_RETRY_ON_TIMEOUT")
|
| 96 |
+
redis_health_check_interval: int = Field(default=30, alias="REDIS_HEALTH_CHECK_INTERVAL")
|
| 97 |
+
|
| 98 |
+
# Scheduler settings
|
| 99 |
+
max_http_timeout: float = 300.0
|
| 100 |
+
default_scheduler_timezone: str = "UTC"
|
| 101 |
+
history_retention_days: int = 30
|
| 102 |
+
ssrf_protection: bool = True
|
| 103 |
+
scheduler_instance_id: str | None = Field(default=None, alias="SCHEDULER_INSTANCE_ID")
|
| 104 |
+
scheduler_lock_timeout: int = Field(default=300, alias="SCHEDULER_LOCK_TIMEOUT")
|
| 105 |
+
scheduler_misfire_grace_time: int = Field(default=300, alias="SCHEDULER_MISFIRE_GRACE_TIME")
|
| 106 |
+
scheduler_coordinator_prefix: str = Field(default="scheduler:", alias="SCHEDULER_COORDINATOR_PREFIX")
|
| 107 |
+
|
| 108 |
@property
|
| 109 |
def max_upload_mb(self) -> int:
|
| 110 |
return self.max_upload_bytes // (1024 * 1024)
|
app/services/scheduler_service.py
ADDED
|
@@ -0,0 +1,1358 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import re
|
| 7 |
+
import secrets
|
| 8 |
+
import uuid
|
| 9 |
+
from datetime import UTC, datetime, timedelta
|
| 10 |
+
from enum import Enum
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
from apscheduler.executors.asyncio import AsyncIOExecutor
|
| 15 |
+
from apscheduler.jobstores.memory import MemoryJobStore
|
| 16 |
+
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
| 17 |
+
from apscheduler.triggers.cron import CronTrigger
|
| 18 |
+
from apscheduler.triggers.date import DateTrigger
|
| 19 |
+
from apscheduler.triggers.interval import IntervalTrigger
|
| 20 |
+
from croniter import croniter
|
| 21 |
+
|
| 22 |
+
from app.config import get_settings
|
| 23 |
+
from app.core.logger import get_logger
|
| 24 |
+
from app.services.supabase import SupabaseClient, get_supabase_client
|
| 25 |
+
|
| 26 |
+
logger = get_logger(__name__)
|
| 27 |
+
settings = get_settings()
|
| 28 |
+
|
| 29 |
+
# ---------------------------------------------------------------------------
|
| 30 |
+
# Redis lazy import helper
|
| 31 |
+
# ---------------------------------------------------------------------------
|
| 32 |
+
|
| 33 |
+
_redis_imported: bool = False
|
| 34 |
+
_redis_asyncio: Any = None
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _get_redis_asyncio():
|
| 38 |
+
global _redis_imported, _redis_asyncio
|
| 39 |
+
if not _redis_imported:
|
| 40 |
+
try:
|
| 41 |
+
import redis.asyncio as ra
|
| 42 |
+
_redis_asyncio = ra
|
| 43 |
+
except ImportError:
|
| 44 |
+
_redis_asyncio = None
|
| 45 |
+
_redis_imported = True
|
| 46 |
+
return _redis_asyncio
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _get_redis_jobstore_cls():
|
| 50 |
+
try:
|
| 51 |
+
from apscheduler.jobstores.redis import RedisJobStore
|
| 52 |
+
return RedisJobStore
|
| 53 |
+
except ImportError:
|
| 54 |
+
return None
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ---------------------------------------------------------------------------
|
| 58 |
+
# Enumerations
|
| 59 |
+
# ---------------------------------------------------------------------------
|
| 60 |
+
|
| 61 |
+
class JobStatus(str, Enum):
|
| 62 |
+
ACTIVE = "active"
|
| 63 |
+
PAUSED = "paused"
|
| 64 |
+
COMPLETED = "completed"
|
| 65 |
+
FAILED = "failed"
|
| 66 |
+
DELETED = "deleted"
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class TriggerType(str, Enum):
|
| 70 |
+
CRON = "cron"
|
| 71 |
+
INTERVAL = "interval"
|
| 72 |
+
DATE = "date"
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class HttpMethod(str, Enum):
|
| 76 |
+
GET = "GET"
|
| 77 |
+
POST = "POST"
|
| 78 |
+
PUT = "PUT"
|
| 79 |
+
PATCH = "PATCH"
|
| 80 |
+
DELETE = "DELETE"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class AuthType(str, Enum):
|
| 84 |
+
NONE = "none"
|
| 85 |
+
BEARER = "bearer"
|
| 86 |
+
API_KEY = "api_key"
|
| 87 |
+
BASIC = "basic"
|
| 88 |
+
CUSTOM = "custom"
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class ExecutionStatus(str, Enum):
|
| 92 |
+
SUCCESS = "success"
|
| 93 |
+
FAILURE = "failure"
|
| 94 |
+
TIMEOUT = "timeout"
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class TriggerReason(str, Enum):
|
| 98 |
+
SCHEDULED = "scheduled"
|
| 99 |
+
MANUAL = "manual"
|
| 100 |
+
RETRY = "retry"
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
# ---------------------------------------------------------------------------
|
| 104 |
+
# SSRF Protection
|
| 105 |
+
# ---------------------------------------------------------------------------
|
| 106 |
+
|
| 107 |
+
_PRIVATE_NETWORKS = [
|
| 108 |
+
re.compile(r"^10\."),
|
| 109 |
+
re.compile(r"^172\.(1[6-9]|2\d|3[01])\."),
|
| 110 |
+
re.compile(r"^192\.168\."),
|
| 111 |
+
re.compile(r"^127\."),
|
| 112 |
+
re.compile(r"^0\."),
|
| 113 |
+
re.compile(r"^169\.254\."),
|
| 114 |
+
re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\."),
|
| 115 |
+
re.compile(r"^fc[0-9a-f]{2}:", re.IGNORECASE),
|
| 116 |
+
re.compile(r"^fe80:", re.IGNORECASE),
|
| 117 |
+
]
|
| 118 |
+
|
| 119 |
+
_BLOCKED_HOSTS: set[str] = {
|
| 120 |
+
"localhost", "127.0.0.1", "0.0.0.0", "::1",
|
| 121 |
+
"169.254.169.254",
|
| 122 |
+
"metadata.google.internal",
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _check_ssrf(url: str) -> None:
|
| 127 |
+
if not settings.ssrf_protection:
|
| 128 |
+
return
|
| 129 |
+
from urllib.parse import urlparse
|
| 130 |
+
parsed = urlparse(url)
|
| 131 |
+
host = parsed.hostname or ""
|
| 132 |
+
if host.lower() in _BLOCKED_HOSTS:
|
| 133 |
+
raise ValueError(f"SSRF protection: host '{host}' is blocked")
|
| 134 |
+
for pattern in _PRIVATE_NETWORKS:
|
| 135 |
+
if pattern.match(host):
|
| 136 |
+
raise ValueError(f"SSRF protection: host '{host}' resolves to a private network")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
# ---------------------------------------------------------------------------
|
| 140 |
+
# Auth Header Builder
|
| 141 |
+
# ---------------------------------------------------------------------------
|
| 142 |
+
|
| 143 |
+
def _build_auth_headers(auth_config: dict[str, Any]) -> dict[str, str]:
|
| 144 |
+
auth_headers: dict[str, str] = {}
|
| 145 |
+
auth_type = auth_config.get("type", "none")
|
| 146 |
+
if auth_type == "bearer":
|
| 147 |
+
auth_headers["Authorization"] = f"Bearer {auth_config.get('token', '')}"
|
| 148 |
+
elif auth_type == "api_key":
|
| 149 |
+
header_name = auth_config.get("api_key_header", "X-API-Key")
|
| 150 |
+
auth_headers[header_name] = auth_config.get("api_key", "")
|
| 151 |
+
elif auth_type == "basic":
|
| 152 |
+
import base64
|
| 153 |
+
username = auth_config.get("username", "")
|
| 154 |
+
password = auth_config.get("password", "")
|
| 155 |
+
encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
|
| 156 |
+
auth_headers["Authorization"] = f"Basic {encoded}"
|
| 157 |
+
elif auth_type == "custom":
|
| 158 |
+
custom = auth_config.get("custom_headers") or {}
|
| 159 |
+
auth_headers.update(custom)
|
| 160 |
+
return auth_headers
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# ---------------------------------------------------------------------------
|
| 164 |
+
# Scheduler Repository (Supabase-backed)
|
| 165 |
+
# ---------------------------------------------------------------------------
|
| 166 |
+
|
| 167 |
+
class SchedulerRepository:
|
| 168 |
+
def __init__(self, client: SupabaseClient):
|
| 169 |
+
self._client = client
|
| 170 |
+
|
| 171 |
+
async def create_job(self, data: dict[str, Any]) -> dict[str, Any]:
|
| 172 |
+
result = await self._client.insert("scheduled_jobs", data)
|
| 173 |
+
return result or {}
|
| 174 |
+
|
| 175 |
+
async def get_job_by_id(self, job_id: str) -> dict[str, Any] | None:
|
| 176 |
+
try:
|
| 177 |
+
return await self._client.find_one("scheduled_jobs", "id", job_id)
|
| 178 |
+
except Exception:
|
| 179 |
+
return None
|
| 180 |
+
|
| 181 |
+
async def get_job_by_name(self, name: str) -> dict[str, Any] | None:
|
| 182 |
+
try:
|
| 183 |
+
return await self._client.find_one("scheduled_jobs", "name", name)
|
| 184 |
+
except Exception:
|
| 185 |
+
return None
|
| 186 |
+
|
| 187 |
+
async def list_jobs(
|
| 188 |
+
self,
|
| 189 |
+
status: str | None = None,
|
| 190 |
+
tags: list[str] | None = None,
|
| 191 |
+
page: int = 1,
|
| 192 |
+
page_size: int = 20,
|
| 193 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 194 |
+
offset = (page - 1) * page_size
|
| 195 |
+
if status and tags:
|
| 196 |
+
rows = await self._client.select("scheduled_jobs", eq=("status", status), limit=page_size, offset=offset)
|
| 197 |
+
filtered = [r for r in rows if tags and any(t in (r.get("tags") or []) for t in tags)]
|
| 198 |
+
total = len(filtered)
|
| 199 |
+
return filtered[:page_size], total
|
| 200 |
+
elif status:
|
| 201 |
+
rows = await self._client.select("scheduled_jobs", eq=("status", status), limit=page_size, offset=offset)
|
| 202 |
+
total = len(await self._client.select("scheduled_jobs", eq=("status", status)))
|
| 203 |
+
return rows, total
|
| 204 |
+
else:
|
| 205 |
+
rows = await self._client.select("scheduled_jobs", limit=page_size, offset=offset)
|
| 206 |
+
total = len(await self._client.select("scheduled_jobs"))
|
| 207 |
+
return rows, total
|
| 208 |
+
|
| 209 |
+
async def update_job(self, job_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
| 210 |
+
data["updated_at"] = datetime.now(UTC).isoformat()
|
| 211 |
+
result = await self._client.update("scheduled_jobs", "id", job_id, data)
|
| 212 |
+
return result[0] if result else None
|
| 213 |
+
|
| 214 |
+
async def delete_job(self, job_id: str) -> None:
|
| 215 |
+
await self._client.delete("scheduled_jobs", "id", job_id)
|
| 216 |
+
|
| 217 |
+
async def get_active_jobs(self) -> list[dict[str, Any]]:
|
| 218 |
+
rows = await self._client.select("scheduled_jobs", eq=("status", JobStatus.ACTIVE.value))
|
| 219 |
+
paused = await self._client.select("scheduled_jobs", eq=("status", JobStatus.PAUSED.value))
|
| 220 |
+
return rows + paused
|
| 221 |
+
|
| 222 |
+
async def create_history(self, data: dict[str, Any]) -> dict[str, Any]:
|
| 223 |
+
result = await self._client.insert("job_execution_history", data)
|
| 224 |
+
return result or {}
|
| 225 |
+
|
| 226 |
+
async def get_history(
|
| 227 |
+
self,
|
| 228 |
+
job_id: str,
|
| 229 |
+
page: int = 1,
|
| 230 |
+
page_size: int = 20,
|
| 231 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 232 |
+
offset = (page - 1) * page_size
|
| 233 |
+
rows = await self._client.select(
|
| 234 |
+
"job_execution_history", eq=("job_id", job_id),
|
| 235 |
+
order=("started_at", True),
|
| 236 |
+
limit=page_size, offset=offset,
|
| 237 |
+
)
|
| 238 |
+
total = await self._client.count("job_execution_history", "job_id", job_id)
|
| 239 |
+
return rows, total
|
| 240 |
+
|
| 241 |
+
async def get_history_all(
|
| 242 |
+
self,
|
| 243 |
+
page: int = 1,
|
| 244 |
+
page_size: int = 50,
|
| 245 |
+
status_filter: str | None = None,
|
| 246 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 247 |
+
offset = (page - 1) * page_size
|
| 248 |
+
if status_filter:
|
| 249 |
+
rows = await self._client.select(
|
| 250 |
+
"job_execution_history", eq=("status", status_filter),
|
| 251 |
+
order=("started_at", True),
|
| 252 |
+
limit=page_size, offset=offset,
|
| 253 |
+
)
|
| 254 |
+
total = await self._client.count("job_execution_history", "status", status_filter)
|
| 255 |
+
else:
|
| 256 |
+
rows = await self._client.select(
|
| 257 |
+
"job_execution_history",
|
| 258 |
+
order=("started_at", True),
|
| 259 |
+
limit=page_size, offset=offset,
|
| 260 |
+
)
|
| 261 |
+
total = len(await self._client.select("job_execution_history"))
|
| 262 |
+
return rows, total
|
| 263 |
+
|
| 264 |
+
async def purge_old_history(self, days: int) -> int:
|
| 265 |
+
cutoff = (datetime.now(UTC) - timedelta(days=days)).isoformat()
|
| 266 |
+
rows = await self._client.select("job_execution_history")
|
| 267 |
+
deleted = 0
|
| 268 |
+
for row in rows:
|
| 269 |
+
created = row.get("created_at", "")
|
| 270 |
+
if created < cutoff:
|
| 271 |
+
await self._client.delete("job_execution_history", "id", row["id"])
|
| 272 |
+
deleted += 1
|
| 273 |
+
return deleted
|
| 274 |
+
|
| 275 |
+
async def count_jobs(self, status: str | None = None) -> int:
|
| 276 |
+
if status:
|
| 277 |
+
return await self._client.count("scheduled_jobs", "status", status)
|
| 278 |
+
rows = await self._client.select("scheduled_jobs")
|
| 279 |
+
return len(rows)
|
| 280 |
+
|
| 281 |
+
async def count_executions(self, status: str | None = None) -> int:
|
| 282 |
+
if status:
|
| 283 |
+
return await self._client.count("job_execution_history", "status", status)
|
| 284 |
+
rows = await self._client.select("job_execution_history")
|
| 285 |
+
return len(rows)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
# ---------------------------------------------------------------------------
|
| 289 |
+
# HTTP Execution Engine
|
| 290 |
+
# ---------------------------------------------------------------------------
|
| 291 |
+
|
| 292 |
+
class HttpExecutionResult:
|
| 293 |
+
__slots__ = (
|
| 294 |
+
"success", "status_code", "response_body",
|
| 295 |
+
"response_size", "duration_ms", "error_message",
|
| 296 |
+
"exception_type", "retry_count",
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
def __init__(
|
| 300 |
+
self,
|
| 301 |
+
success: bool,
|
| 302 |
+
status_code: int | None = None,
|
| 303 |
+
response_body: str | None = None,
|
| 304 |
+
response_size: int = 0,
|
| 305 |
+
duration_ms: float = 0.0,
|
| 306 |
+
error_message: str | None = None,
|
| 307 |
+
exception_type: str | None = None,
|
| 308 |
+
retry_count: int = 0,
|
| 309 |
+
) -> None:
|
| 310 |
+
self.success = success
|
| 311 |
+
self.status_code = status_code
|
| 312 |
+
self.response_body = response_body
|
| 313 |
+
self.response_size = response_size
|
| 314 |
+
self.duration_ms = duration_ms
|
| 315 |
+
self.error_message = error_message
|
| 316 |
+
self.exception_type = exception_type
|
| 317 |
+
self.retry_count = retry_count
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
class HttpExecutionEngine:
|
| 321 |
+
_MAX_RESPONSE_PREVIEW = 500
|
| 322 |
+
|
| 323 |
+
def __init__(self) -> None:
|
| 324 |
+
self._client: httpx.AsyncClient | None = None
|
| 325 |
+
self._lock = asyncio.Lock()
|
| 326 |
+
|
| 327 |
+
async def _get_client(self) -> httpx.AsyncClient:
|
| 328 |
+
if self._client is None or self._client.is_closed:
|
| 329 |
+
async with self._lock:
|
| 330 |
+
if self._client is None or self._client.is_closed:
|
| 331 |
+
self._client = httpx.AsyncClient(
|
| 332 |
+
follow_redirects=True,
|
| 333 |
+
limits=httpx.Limits(
|
| 334 |
+
max_connections=200,
|
| 335 |
+
max_keepalive_connections=50,
|
| 336 |
+
keepalive_expiry=30,
|
| 337 |
+
),
|
| 338 |
+
timeout=httpx.Timeout(settings.max_http_timeout),
|
| 339 |
+
)
|
| 340 |
+
return self._client
|
| 341 |
+
|
| 342 |
+
async def close(self) -> None:
|
| 343 |
+
if self._client and not self._client.is_closed:
|
| 344 |
+
await self._client.aclose()
|
| 345 |
+
logger.info("HTTP client closed")
|
| 346 |
+
|
| 347 |
+
async def execute(self, job: dict[str, Any]) -> HttpExecutionResult:
|
| 348 |
+
retry_on_status: list[int] = job.get("retry_on_status") or [429, 500, 502, 503, 504]
|
| 349 |
+
max_retries: int = job.get("max_retries", 3)
|
| 350 |
+
retry_delay: float = job.get("retry_delay_seconds", 1.0)
|
| 351 |
+
backoff: float = job.get("retry_backoff", 2.0)
|
| 352 |
+
max_delay: float = job.get("retry_max_delay_seconds", 60.0)
|
| 353 |
+
retry_jitter: bool = job.get("retry_jitter", False)
|
| 354 |
+
|
| 355 |
+
last_result: HttpExecutionResult | None = None
|
| 356 |
+
|
| 357 |
+
for attempt in range(max_retries + 1):
|
| 358 |
+
if attempt > 0:
|
| 359 |
+
delay = min(retry_delay * (backoff ** (attempt - 1)), max_delay)
|
| 360 |
+
if retry_jitter:
|
| 361 |
+
delay = delay * (0.5 + secrets.randbelow(1000) / 1000.0)
|
| 362 |
+
logger.info("Job retry wait", extra={
|
| 363 |
+
"job_id": job["id"], "job_name": job["name"],
|
| 364 |
+
"attempt": attempt, "delay_seconds": round(delay, 2),
|
| 365 |
+
})
|
| 366 |
+
await asyncio.sleep(delay)
|
| 367 |
+
|
| 368 |
+
result = await self._execute_once(job, attempt)
|
| 369 |
+
|
| 370 |
+
if result.success:
|
| 371 |
+
result.retry_count = attempt
|
| 372 |
+
return result
|
| 373 |
+
|
| 374 |
+
last_result = result
|
| 375 |
+
|
| 376 |
+
should_retry = False
|
| 377 |
+
if attempt < max_retries:
|
| 378 |
+
if result.exception_type == "TimeoutException" and job.get("retry_on_timeout", True):
|
| 379 |
+
should_retry = True
|
| 380 |
+
elif result.exception_type in ("ConnectError", "ReadError", "NetworkError") and job.get("retry_on_failure", True):
|
| 381 |
+
should_retry = True
|
| 382 |
+
elif result.status_code in retry_on_status:
|
| 383 |
+
should_retry = True
|
| 384 |
+
|
| 385 |
+
if not should_retry:
|
| 386 |
+
break
|
| 387 |
+
|
| 388 |
+
logger.warning("Job execution retry", extra={
|
| 389 |
+
"job_id": job["id"], "job_name": job["name"],
|
| 390 |
+
"attempt": attempt + 1, "max_retries": max_retries,
|
| 391 |
+
"status_code": result.status_code, "error": result.error_message,
|
| 392 |
+
})
|
| 393 |
+
|
| 394 |
+
if last_result is not None:
|
| 395 |
+
last_result.retry_count = max(0, max_retries)
|
| 396 |
+
return last_result or HttpExecutionResult(success=False, error_message="Unknown error")
|
| 397 |
+
|
| 398 |
+
async def _execute_once(self, job: dict[str, Any], attempt: int) -> HttpExecutionResult:
|
| 399 |
+
started = datetime.now(UTC)
|
| 400 |
+
client = await self._get_client()
|
| 401 |
+
|
| 402 |
+
headers: dict[str, str] = dict(job.get("headers") or {})
|
| 403 |
+
auth_headers = _build_auth_headers(job.get("auth_config") or {})
|
| 404 |
+
headers.update(auth_headers)
|
| 405 |
+
headers.setdefault("User-Agent", f"{settings.app_name}/{settings.app_version}")
|
| 406 |
+
|
| 407 |
+
params: dict[str, str] = dict(job.get("query_params") or {})
|
| 408 |
+
method = (job.get("method") or "GET").upper()
|
| 409 |
+
url = job.get("url", "")
|
| 410 |
+
|
| 411 |
+
try:
|
| 412 |
+
_check_ssrf(url)
|
| 413 |
+
except ValueError as exc:
|
| 414 |
+
return HttpExecutionResult(
|
| 415 |
+
success=False, error_message=str(exc), exception_type="SSRFViolation",
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
request_kwargs: dict[str, Any] = {
|
| 419 |
+
"method": method,
|
| 420 |
+
"url": url,
|
| 421 |
+
"headers": headers,
|
| 422 |
+
"params": params,
|
| 423 |
+
"timeout": job.get("timeout_seconds", 30.0),
|
| 424 |
+
}
|
| 425 |
+
|
| 426 |
+
body = job.get("body")
|
| 427 |
+
if body is not None and method in ("POST", "PUT", "PATCH", "DELETE"):
|
| 428 |
+
body_type = job.get("body_type", "json")
|
| 429 |
+
if body_type == "json":
|
| 430 |
+
request_kwargs["json"] = body
|
| 431 |
+
elif body_type == "form":
|
| 432 |
+
request_kwargs["data"] = body
|
| 433 |
+
elif body_type == "raw":
|
| 434 |
+
request_kwargs["content"] = str(body).encode()
|
| 435 |
+
|
| 436 |
+
try:
|
| 437 |
+
response = await client.request(**request_kwargs)
|
| 438 |
+
duration_ms = (datetime.now(UTC) - started).total_seconds() * 1000
|
| 439 |
+
response_body = response.text[:self._MAX_RESPONSE_PREVIEW]
|
| 440 |
+
success = response.is_success
|
| 441 |
+
|
| 442 |
+
if not success:
|
| 443 |
+
logger.warning("HTTP non-success status", extra={
|
| 444 |
+
"job_id": job["id"], "job_name": job["name"],
|
| 445 |
+
"status_code": response.status_code, "attempt": attempt,
|
| 446 |
+
})
|
| 447 |
+
|
| 448 |
+
return HttpExecutionResult(
|
| 449 |
+
success=success,
|
| 450 |
+
status_code=response.status_code,
|
| 451 |
+
response_body=response_body,
|
| 452 |
+
response_size=len(response.content),
|
| 453 |
+
duration_ms=duration_ms,
|
| 454 |
+
error_message=None if success else f"HTTP {response.status_code}",
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
except httpx.TimeoutException as exc:
|
| 458 |
+
duration_ms = (datetime.now(UTC) - started).total_seconds() * 1000
|
| 459 |
+
return HttpExecutionResult(
|
| 460 |
+
success=False, duration_ms=duration_ms,
|
| 461 |
+
error_message=f"Request timed out after {job.get('timeout_seconds', 30.0)}s",
|
| 462 |
+
exception_type="TimeoutException",
|
| 463 |
+
)
|
| 464 |
+
except httpx.ConnectError as exc:
|
| 465 |
+
return HttpExecutionResult(
|
| 466 |
+
success=False, error_message=f"Connection error: {exc}",
|
| 467 |
+
exception_type="ConnectError",
|
| 468 |
+
)
|
| 469 |
+
except httpx.ReadError as exc:
|
| 470 |
+
return HttpExecutionResult(
|
| 471 |
+
success=False, error_message=f"Read error: {exc}",
|
| 472 |
+
exception_type="ReadError",
|
| 473 |
+
)
|
| 474 |
+
except Exception as exc:
|
| 475 |
+
logger.error("HTTP unexpected error", extra={
|
| 476 |
+
"job_id": job["id"], "job_name": job["name"],
|
| 477 |
+
"attempt": attempt, "error": str(exc),
|
| 478 |
+
})
|
| 479 |
+
return HttpExecutionResult(
|
| 480 |
+
success=False, error_message=str(exc),
|
| 481 |
+
exception_type=type(exc).__name__,
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
# ---------------------------------------------------------------------------
|
| 486 |
+
# Scheduler Service — Production-Grade with Redis HA
|
| 487 |
+
# ---------------------------------------------------------------------------
|
| 488 |
+
|
| 489 |
+
class SchedulerService:
|
| 490 |
+
def __init__(self) -> None:
|
| 491 |
+
self._scheduler: AsyncIOScheduler | None = None
|
| 492 |
+
self._async_redis: Any = None
|
| 493 |
+
self._use_redis = False
|
| 494 |
+
self._instance_id: str = settings.scheduler_instance_id or str(uuid.uuid4())[:8]
|
| 495 |
+
self._http_engine = HttpExecutionEngine()
|
| 496 |
+
self._running_jobs: dict[str, asyncio.Task[None]] = {}
|
| 497 |
+
self._lock = asyncio.Lock()
|
| 498 |
+
self._maintenance_task: asyncio.Task[None] | None = None
|
| 499 |
+
self._repo: SchedulerRepository | None = None
|
| 500 |
+
|
| 501 |
+
# -----------------------------------------------------------------------
|
| 502 |
+
# Redis connection management
|
| 503 |
+
# -----------------------------------------------------------------------
|
| 504 |
+
|
| 505 |
+
async def _connect_async_redis(self) -> Any:
|
| 506 |
+
ra = _get_redis_asyncio()
|
| 507 |
+
if ra is None:
|
| 508 |
+
return None
|
| 509 |
+
try:
|
| 510 |
+
client = ra.Redis(
|
| 511 |
+
host=settings.redis_host,
|
| 512 |
+
port=settings.redis_port,
|
| 513 |
+
password=settings.redis_password or None,
|
| 514 |
+
db=settings.redis_db,
|
| 515 |
+
ssl=settings.redis_ssl,
|
| 516 |
+
socket_timeout=settings.redis_socket_timeout,
|
| 517 |
+
socket_connect_timeout=settings.redis_socket_connect_timeout,
|
| 518 |
+
retry_on_timeout=settings.redis_retry_on_timeout,
|
| 519 |
+
health_check_interval=settings.redis_health_check_interval,
|
| 520 |
+
decode_responses=True,
|
| 521 |
+
)
|
| 522 |
+
await client.ping()
|
| 523 |
+
logger.info(
|
| 524 |
+
"Connected to Redis at %s:%d (db=%d)",
|
| 525 |
+
settings.redis_host, settings.redis_port, settings.redis_db,
|
| 526 |
+
)
|
| 527 |
+
return client
|
| 528 |
+
except Exception as exc:
|
| 529 |
+
logger.warning("Redis connection failed: %s — falling back to in-memory scheduler", exc)
|
| 530 |
+
return None
|
| 531 |
+
|
| 532 |
+
def _create_redis_jobstore(self) -> Any:
|
| 533 |
+
cls = _get_redis_jobstore_cls()
|
| 534 |
+
if cls is None:
|
| 535 |
+
raise RuntimeError("apscheduler[jobstores_redis] not installed")
|
| 536 |
+
return cls(
|
| 537 |
+
db=settings.redis_db,
|
| 538 |
+
host=settings.redis_host,
|
| 539 |
+
port=settings.redis_port,
|
| 540 |
+
password=settings.redis_password or None,
|
| 541 |
+
ssl=settings.redis_ssl,
|
| 542 |
+
socket_timeout=settings.redis_socket_timeout,
|
| 543 |
+
socket_connect_timeout=settings.redis_socket_connect_timeout,
|
| 544 |
+
retry_on_timeout=settings.redis_retry_on_timeout,
|
| 545 |
+
health_check_interval=settings.redis_health_check_interval,
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
# -----------------------------------------------------------------------
|
| 549 |
+
# Distributed execution lock (Redis SETNX)
|
| 550 |
+
# -----------------------------------------------------------------------
|
| 551 |
+
|
| 552 |
+
def _lock_key(self, job_id: str) -> str:
|
| 553 |
+
return f"{settings.scheduler_coordinator_prefix}lock:{job_id}"
|
| 554 |
+
|
| 555 |
+
async def _acquire_execution_lock(self, job: dict[str, Any]) -> bool:
|
| 556 |
+
if not self._use_redis or self._async_redis is None:
|
| 557 |
+
return True
|
| 558 |
+
key = self._lock_key(job["id"])
|
| 559 |
+
ttl = job.get("timeout_seconds", 30) + 30
|
| 560 |
+
acquired = await self._async_redis.setnx(key, self._instance_id)
|
| 561 |
+
if acquired:
|
| 562 |
+
await self._async_redis.expire(key, int(ttl))
|
| 563 |
+
logger.debug("Acquired execution lock for job %s", job["id"])
|
| 564 |
+
return True
|
| 565 |
+
owner = await self._async_redis.get(key)
|
| 566 |
+
logger.warning(
|
| 567 |
+
"Execution lock held by instance %s for job %s — skipping",
|
| 568 |
+
owner, job["id"],
|
| 569 |
+
)
|
| 570 |
+
return False
|
| 571 |
+
|
| 572 |
+
async def _release_execution_lock(self, job_id: str) -> None:
|
| 573 |
+
if not self._use_redis or self._async_redis is None:
|
| 574 |
+
return
|
| 575 |
+
key = self._lock_key(job_id)
|
| 576 |
+
await self._async_redis.delete(key)
|
| 577 |
+
logger.debug("Released execution lock for job %s", job_id)
|
| 578 |
+
|
| 579 |
+
# -----------------------------------------------------------------------
|
| 580 |
+
# Redis health
|
| 581 |
+
# -----------------------------------------------------------------------
|
| 582 |
+
|
| 583 |
+
async def _check_redis_health(self) -> dict[str, Any]:
|
| 584 |
+
if not self._use_redis or self._async_redis is None:
|
| 585 |
+
return {"connected": False, "mode": "memory"}
|
| 586 |
+
try:
|
| 587 |
+
ping = await self._async_redis.ping()
|
| 588 |
+
info = await self._async_redis.info(section="server")
|
| 589 |
+
return {
|
| 590 |
+
"connected": bool(ping),
|
| 591 |
+
"mode": "redis",
|
| 592 |
+
"redis_version": info.get("redis_version", "unknown"),
|
| 593 |
+
"instance_id": self._instance_id,
|
| 594 |
+
}
|
| 595 |
+
except Exception as exc:
|
| 596 |
+
return {"connected": False, "mode": "redis", "error": str(exc)}
|
| 597 |
+
|
| 598 |
+
# -----------------------------------------------------------------------
|
| 599 |
+
# Lifecycle
|
| 600 |
+
# -----------------------------------------------------------------------
|
| 601 |
+
|
| 602 |
+
async def start(self) -> None:
|
| 603 |
+
self._async_redis = await self._connect_async_redis()
|
| 604 |
+
self._use_redis = self._async_redis is not None
|
| 605 |
+
|
| 606 |
+
jobstores: dict[str, Any] = {}
|
| 607 |
+
if self._use_redis:
|
| 608 |
+
try:
|
| 609 |
+
jobstores["default"] = self._create_redis_jobstore()
|
| 610 |
+
logger.info("Using RedisJobStore for job persistence")
|
| 611 |
+
except Exception as exc:
|
| 612 |
+
logger.warning("Failed to create RedisJobStore: %s — falling back to memory", exc)
|
| 613 |
+
self._use_redis = False
|
| 614 |
+
self._async_redis = None
|
| 615 |
+
|
| 616 |
+
if not self._use_redis:
|
| 617 |
+
jobstores["default"] = MemoryJobStore()
|
| 618 |
+
logger.info("Using MemoryJobStore (jobs will not survive restart)")
|
| 619 |
+
|
| 620 |
+
executors = {"default": AsyncIOExecutor()}
|
| 621 |
+
job_defaults = {
|
| 622 |
+
"coalesce": True,
|
| 623 |
+
"max_instances": 1,
|
| 624 |
+
"misfire_grace_time": settings.scheduler_misfire_grace_time,
|
| 625 |
+
}
|
| 626 |
+
|
| 627 |
+
self._scheduler = AsyncIOScheduler(
|
| 628 |
+
jobstores=jobstores,
|
| 629 |
+
executors=executors,
|
| 630 |
+
job_defaults=job_defaults,
|
| 631 |
+
)
|
| 632 |
+
self._scheduler.start()
|
| 633 |
+
logger.info("Scheduler started (mode=%s, instance=%s)", "redis" if self._use_redis else "memory", self._instance_id)
|
| 634 |
+
|
| 635 |
+
try:
|
| 636 |
+
client = get_supabase_client()
|
| 637 |
+
if client:
|
| 638 |
+
repo = SchedulerRepository(client)
|
| 639 |
+
jobs = await repo.get_active_jobs()
|
| 640 |
+
restored = 0
|
| 641 |
+
for job in jobs:
|
| 642 |
+
try:
|
| 643 |
+
self._schedule_job(job)
|
| 644 |
+
restored += 1
|
| 645 |
+
except Exception as exc:
|
| 646 |
+
logger.error("Failed to restore job %s: %s", job.get("id"), exc)
|
| 647 |
+
|
| 648 |
+
if self._use_redis:
|
| 649 |
+
aps_jobs = self._scheduler.get_jobs()
|
| 650 |
+
aps_ids = {j.id for j in aps_jobs}
|
| 651 |
+
db_ids = {f"job_{j['id']}" for j in jobs}
|
| 652 |
+
stale = aps_ids - db_ids
|
| 653 |
+
for sid in stale:
|
| 654 |
+
try:
|
| 655 |
+
self._scheduler.remove_job(sid)
|
| 656 |
+
logger.info("Removed stale job %s from Redis store", sid)
|
| 657 |
+
except Exception:
|
| 658 |
+
pass
|
| 659 |
+
|
| 660 |
+
logger.info("Restored %d jobs from database", restored)
|
| 661 |
+
except Exception as exc:
|
| 662 |
+
logger.warning("Could not restore jobs from database: %s", exc)
|
| 663 |
+
|
| 664 |
+
self._maintenance_task = asyncio.create_task(self._maintenance_loop())
|
| 665 |
+
|
| 666 |
+
async def shutdown(self) -> None:
|
| 667 |
+
logger.info("Scheduler shutting down (instance=%s)", self._instance_id)
|
| 668 |
+
if self._maintenance_task:
|
| 669 |
+
self._maintenance_task.cancel()
|
| 670 |
+
try:
|
| 671 |
+
await asyncio.wait_for(asyncio.shield(self._maintenance_task), timeout=5.0)
|
| 672 |
+
except (asyncio.CancelledError, asyncio.TimeoutError):
|
| 673 |
+
pass
|
| 674 |
+
|
| 675 |
+
if self._scheduler:
|
| 676 |
+
self._scheduler.shutdown(wait=False)
|
| 677 |
+
|
| 678 |
+
async with self._lock:
|
| 679 |
+
tasks = list(self._running_jobs.values())
|
| 680 |
+
for task in tasks:
|
| 681 |
+
task.cancel()
|
| 682 |
+
try:
|
| 683 |
+
await asyncio.wait_for(asyncio.shield(task), timeout=5.0)
|
| 684 |
+
except (asyncio.CancelledError, asyncio.TimeoutError):
|
| 685 |
+
pass
|
| 686 |
+
|
| 687 |
+
await self._http_engine.close()
|
| 688 |
+
|
| 689 |
+
if self._async_redis:
|
| 690 |
+
try:
|
| 691 |
+
await self._async_redis.aclose()
|
| 692 |
+
logger.info("Redis connection closed")
|
| 693 |
+
except Exception:
|
| 694 |
+
pass
|
| 695 |
+
|
| 696 |
+
logger.info("Scheduler shutdown complete")
|
| 697 |
+
|
| 698 |
+
async def _maintenance_loop(self) -> None:
|
| 699 |
+
while True:
|
| 700 |
+
try:
|
| 701 |
+
await asyncio.sleep(3600)
|
| 702 |
+
if self._use_redis and self._async_redis:
|
| 703 |
+
try:
|
| 704 |
+
await self._async_redis.ping()
|
| 705 |
+
except Exception:
|
| 706 |
+
logger.error("Redis ping failed in maintenance loop")
|
| 707 |
+
client = get_supabase_client()
|
| 708 |
+
if client:
|
| 709 |
+
repo = SchedulerRepository(client)
|
| 710 |
+
purged = await repo.purge_old_history(settings.history_retention_days)
|
| 711 |
+
if purged:
|
| 712 |
+
logger.info("Purged %d old history records", purged)
|
| 713 |
+
except asyncio.CancelledError:
|
| 714 |
+
break
|
| 715 |
+
except Exception:
|
| 716 |
+
logger.error("Maintenance loop error", exc_info=True)
|
| 717 |
+
|
| 718 |
+
# -----------------------------------------------------------------------
|
| 719 |
+
# APScheduler helpers
|
| 720 |
+
# -----------------------------------------------------------------------
|
| 721 |
+
|
| 722 |
+
def _build_trigger(self, job: dict[str, Any]):
|
| 723 |
+
tz = job.get("timezone", "UTC")
|
| 724 |
+
cfg = job.get("trigger_config") or {}
|
| 725 |
+
trigger_type = job.get("trigger_type")
|
| 726 |
+
|
| 727 |
+
if trigger_type == TriggerType.CRON.value:
|
| 728 |
+
cron_expr = cfg.get("cron_expression", "* * * * *")
|
| 729 |
+
parts = cron_expr.split()
|
| 730 |
+
return CronTrigger(
|
| 731 |
+
second=parts[0] if len(parts) > 0 else "*",
|
| 732 |
+
minute=parts[1] if len(parts) > 1 else "*",
|
| 733 |
+
hour=parts[2] if len(parts) > 2 else "*",
|
| 734 |
+
day=parts[3] if len(parts) > 3 else "*",
|
| 735 |
+
month=parts[4] if len(parts) > 4 else "*",
|
| 736 |
+
day_of_week=parts[5] if len(parts) > 5 else "*",
|
| 737 |
+
timezone=tz,
|
| 738 |
+
start_date=job.get("start_date"),
|
| 739 |
+
end_date=job.get("end_date"),
|
| 740 |
+
jitter=job.get("jitter_seconds"),
|
| 741 |
+
)
|
| 742 |
+
elif trigger_type == TriggerType.INTERVAL.value:
|
| 743 |
+
return IntervalTrigger(
|
| 744 |
+
weeks=cfg.get("weeks", 0),
|
| 745 |
+
days=cfg.get("days", 0),
|
| 746 |
+
hours=cfg.get("hours", 0),
|
| 747 |
+
minutes=cfg.get("minutes", 0),
|
| 748 |
+
seconds=cfg.get("seconds", 0),
|
| 749 |
+
timezone=tz,
|
| 750 |
+
start_date=job.get("start_date"),
|
| 751 |
+
end_date=job.get("end_date"),
|
| 752 |
+
jitter=job.get("jitter_seconds"),
|
| 753 |
+
)
|
| 754 |
+
elif trigger_type == TriggerType.DATE.value:
|
| 755 |
+
run_date = cfg.get("run_date")
|
| 756 |
+
return DateTrigger(run_date=run_date, timezone=tz)
|
| 757 |
+
raise ValueError(f"Unknown trigger type: {trigger_type}")
|
| 758 |
+
|
| 759 |
+
def _schedule_job(self, job: dict[str, Any]) -> None:
|
| 760 |
+
if self._scheduler is None:
|
| 761 |
+
raise RuntimeError("Scheduler not started")
|
| 762 |
+
trigger = self._build_trigger(job)
|
| 763 |
+
aps_id = f"job_{job['id']}"
|
| 764 |
+
self._scheduler.add_job(
|
| 765 |
+
func=self._execute_job_wrapper,
|
| 766 |
+
trigger=trigger,
|
| 767 |
+
args=[job["id"]],
|
| 768 |
+
id=aps_id,
|
| 769 |
+
name=job.get("name", "unknown"),
|
| 770 |
+
coalesce=job.get("coalesce", True),
|
| 771 |
+
max_instances=job.get("max_instances", 1),
|
| 772 |
+
misfire_grace_time=job.get("misfire_grace_time") or settings.scheduler_misfire_grace_time,
|
| 773 |
+
replace_existing=True,
|
| 774 |
+
)
|
| 775 |
+
logger.info("Scheduled job %s (%s)", job["name"], job["id"])
|
| 776 |
+
|
| 777 |
+
def _reschedule_job(self, job: dict[str, Any]) -> None:
|
| 778 |
+
aps_id = f"job_{job['id']}"
|
| 779 |
+
try:
|
| 780 |
+
self._scheduler.remove_job(aps_id)
|
| 781 |
+
except Exception:
|
| 782 |
+
pass
|
| 783 |
+
if job.get("status") in (JobStatus.ACTIVE.value, JobStatus.PAUSED.value):
|
| 784 |
+
self._schedule_job(job)
|
| 785 |
+
if job.get("status") == JobStatus.PAUSED.value:
|
| 786 |
+
try:
|
| 787 |
+
self._scheduler.pause_job(aps_id)
|
| 788 |
+
except Exception:
|
| 789 |
+
pass
|
| 790 |
+
|
| 791 |
+
def _remove_aps_job(self, job_id: str) -> None:
|
| 792 |
+
aps_id = f"job_{job_id}"
|
| 793 |
+
try:
|
| 794 |
+
self._scheduler.remove_job(aps_id)
|
| 795 |
+
except Exception:
|
| 796 |
+
pass
|
| 797 |
+
|
| 798 |
+
def pause_aps_job(self, job_id: str) -> None:
|
| 799 |
+
aps_id = f"job_{job_id}"
|
| 800 |
+
try:
|
| 801 |
+
self._scheduler.pause_job(aps_id)
|
| 802 |
+
except Exception as exc:
|
| 803 |
+
raise RuntimeError(f"Failed to pause job: {exc}") from exc
|
| 804 |
+
|
| 805 |
+
def resume_aps_job(self, job_id: str) -> None:
|
| 806 |
+
aps_id = f"job_{job_id}"
|
| 807 |
+
try:
|
| 808 |
+
self._scheduler.resume_job(aps_id)
|
| 809 |
+
except Exception as exc:
|
| 810 |
+
raise RuntimeError(f"Failed to resume job: {exc}") from exc
|
| 811 |
+
|
| 812 |
+
def get_next_run_time(self, job_id: str) -> datetime | None:
|
| 813 |
+
if self._scheduler is None:
|
| 814 |
+
return None
|
| 815 |
+
aps_id = f"job_{job_id}"
|
| 816 |
+
aps_job = self._scheduler.get_job(aps_id)
|
| 817 |
+
return aps_job.next_run_time if aps_job else None
|
| 818 |
+
|
| 819 |
+
def get_scheduler_status(self) -> dict[str, Any]:
|
| 820 |
+
if self._scheduler is None:
|
| 821 |
+
return {"running": False, "pending_jobs": 0, "currently_executing": 0}
|
| 822 |
+
return {
|
| 823 |
+
"running": self._scheduler.running,
|
| 824 |
+
"pending_jobs": len(self._scheduler.get_jobs()),
|
| 825 |
+
"currently_executing": len(self._running_jobs),
|
| 826 |
+
}
|
| 827 |
+
|
| 828 |
+
async def get_health(self) -> dict[str, Any]:
|
| 829 |
+
redis_status = await self._check_redis_health()
|
| 830 |
+
scheduler_status = self.get_scheduler_status()
|
| 831 |
+
return {
|
| 832 |
+
"scheduler": scheduler_status,
|
| 833 |
+
"redis": redis_status,
|
| 834 |
+
"instance_id": self._instance_id,
|
| 835 |
+
"mode": "redis" if self._use_redis else "memory",
|
| 836 |
+
"running_job_ids": self.get_running_job_ids(),
|
| 837 |
+
}
|
| 838 |
+
|
| 839 |
+
def get_running_job_ids(self) -> list[str]:
|
| 840 |
+
return list(self._running_jobs.keys())
|
| 841 |
+
|
| 842 |
+
async def _execute_job_wrapper(self, job_id: str) -> None:
|
| 843 |
+
task = asyncio.create_task(self._execute_job(job_id))
|
| 844 |
+
async with self._lock:
|
| 845 |
+
self._running_jobs[job_id] = task
|
| 846 |
+
try:
|
| 847 |
+
await task
|
| 848 |
+
except asyncio.CancelledError:
|
| 849 |
+
logger.warning("Job task cancelled: %s", job_id)
|
| 850 |
+
except Exception:
|
| 851 |
+
logger.error("Job task error: %s", job_id, exc_info=True)
|
| 852 |
+
finally:
|
| 853 |
+
async with self._lock:
|
| 854 |
+
self._running_jobs.pop(job_id, None)
|
| 855 |
+
|
| 856 |
+
async def _execute_job(
|
| 857 |
+
self,
|
| 858 |
+
job_id: str,
|
| 859 |
+
trigger_reason: TriggerReason = TriggerReason.SCHEDULED,
|
| 860 |
+
) -> None:
|
| 861 |
+
started_at = datetime.now(UTC)
|
| 862 |
+
logger.info("Job execution started", extra={
|
| 863 |
+
"job_id": job_id, "trigger_reason": trigger_reason.value,
|
| 864 |
+
})
|
| 865 |
+
|
| 866 |
+
client = get_supabase_client()
|
| 867 |
+
if client is None:
|
| 868 |
+
logger.error("Supabase not available, cannot execute job %s", job_id)
|
| 869 |
+
return
|
| 870 |
+
|
| 871 |
+
repo = SchedulerRepository(client)
|
| 872 |
+
job = await repo.get_job_by_id(job_id)
|
| 873 |
+
if not job:
|
| 874 |
+
logger.error("Job not found: %s", job_id)
|
| 875 |
+
return
|
| 876 |
+
|
| 877 |
+
if job.get("status") not in (JobStatus.ACTIVE.value,):
|
| 878 |
+
logger.warning("Job %s not active, skipping", job_id)
|
| 879 |
+
return
|
| 880 |
+
|
| 881 |
+
# Acquire distributed lock (no-op in memory mode)
|
| 882 |
+
if not await self._acquire_execution_lock(job):
|
| 883 |
+
return
|
| 884 |
+
|
| 885 |
+
try:
|
| 886 |
+
# Enforce execution timeout
|
| 887 |
+
timeout = job.get("timeout_seconds", 30.0)
|
| 888 |
+
result = await asyncio.wait_for(
|
| 889 |
+
self._http_engine.execute(job),
|
| 890 |
+
timeout=timeout + 10.0,
|
| 891 |
+
)
|
| 892 |
+
except asyncio.TimeoutError:
|
| 893 |
+
result = HttpExecutionResult(
|
| 894 |
+
success=False,
|
| 895 |
+
error_message=f"Job execution timed out after {timeout + 10.0}s",
|
| 896 |
+
exception_type="ExecutionTimeout",
|
| 897 |
+
)
|
| 898 |
+
except Exception as exc:
|
| 899 |
+
result = HttpExecutionResult(
|
| 900 |
+
success=False,
|
| 901 |
+
error_message=str(exc),
|
| 902 |
+
exception_type=type(exc).__name__,
|
| 903 |
+
)
|
| 904 |
+
finally:
|
| 905 |
+
await self._release_execution_lock(job_id)
|
| 906 |
+
|
| 907 |
+
ended_at = datetime.now(UTC)
|
| 908 |
+
duration_ms = (ended_at - started_at).total_seconds() * 1000
|
| 909 |
+
|
| 910 |
+
exec_status = ExecutionStatus.SUCCESS if result.success else ExecutionStatus.FAILURE
|
| 911 |
+
if result.exception_type in ("TimeoutException", "ExecutionTimeout"):
|
| 912 |
+
exec_status = ExecutionStatus.TIMEOUT
|
| 913 |
+
|
| 914 |
+
headers_hash = hashlib.sha256(
|
| 915 |
+
json.dumps(job.get("headers") or {}, sort_keys=True).encode()
|
| 916 |
+
).hexdigest() if job.get("headers") else ""
|
| 917 |
+
|
| 918 |
+
history_data = {
|
| 919 |
+
"id": str(uuid.uuid4()),
|
| 920 |
+
"job_id": job["id"],
|
| 921 |
+
"job_name": job.get("name", ""),
|
| 922 |
+
"started_at": started_at.isoformat(),
|
| 923 |
+
"ended_at": ended_at.isoformat(),
|
| 924 |
+
"duration_ms": round(duration_ms, 2),
|
| 925 |
+
"status": exec_status.value,
|
| 926 |
+
"http_status_code": result.status_code,
|
| 927 |
+
"response_size_bytes": result.response_size,
|
| 928 |
+
"retry_count": result.retry_count,
|
| 929 |
+
"trigger_reason": trigger_reason.value,
|
| 930 |
+
"error_message": result.error_message,
|
| 931 |
+
"exception_type": result.exception_type,
|
| 932 |
+
"request_url": job.get("url", ""),
|
| 933 |
+
"request_method": job.get("method", "GET"),
|
| 934 |
+
"response_preview": result.response_body,
|
| 935 |
+
}
|
| 936 |
+
await repo.create_history(history_data)
|
| 937 |
+
|
| 938 |
+
update_data: dict[str, Any] = {
|
| 939 |
+
"last_run_at": started_at.isoformat(),
|
| 940 |
+
"run_count": (job.get("run_count") or 0) + 1,
|
| 941 |
+
}
|
| 942 |
+
if result.success:
|
| 943 |
+
update_data["success_count"] = (job.get("success_count") or 0) + 1
|
| 944 |
+
else:
|
| 945 |
+
update_data["failure_count"] = (job.get("failure_count") or 0) + 1
|
| 946 |
+
|
| 947 |
+
next_run = self.get_next_run_time(job_id)
|
| 948 |
+
if next_run:
|
| 949 |
+
update_data["next_run_at"] = next_run.isoformat()
|
| 950 |
+
|
| 951 |
+
await repo.update_job(job_id, update_data)
|
| 952 |
+
|
| 953 |
+
if result.success:
|
| 954 |
+
logger.info("Job execution success", extra={
|
| 955 |
+
"job_id": job_id, "job_name": job.get("name"),
|
| 956 |
+
"duration_ms": round(duration_ms, 2),
|
| 957 |
+
"http_status": result.status_code,
|
| 958 |
+
"response_size": result.response_size,
|
| 959 |
+
"retry_count": result.retry_count,
|
| 960 |
+
})
|
| 961 |
+
else:
|
| 962 |
+
logger.error("Job execution failure", extra={
|
| 963 |
+
"job_id": job_id, "job_name": job.get("name"),
|
| 964 |
+
"duration_ms": round(duration_ms, 2),
|
| 965 |
+
"http_status": result.status_code,
|
| 966 |
+
"error": result.error_message,
|
| 967 |
+
"exception_type": result.exception_type,
|
| 968 |
+
"retry_count": result.retry_count,
|
| 969 |
+
})
|
| 970 |
+
|
| 971 |
+
# -----------------------------------------------------------------------
|
| 972 |
+
# Public API: Job CRUD
|
| 973 |
+
# -----------------------------------------------------------------------
|
| 974 |
+
|
| 975 |
+
async def create_job(self, body: dict[str, Any]) -> dict[str, Any]:
|
| 976 |
+
client = get_supabase_client()
|
| 977 |
+
if client is None:
|
| 978 |
+
raise RuntimeError("Supabase not available")
|
| 979 |
+
|
| 980 |
+
repo = SchedulerRepository(client)
|
| 981 |
+
|
| 982 |
+
existing = await repo.get_job_by_name(body["name"])
|
| 983 |
+
if existing:
|
| 984 |
+
raise ValueError(f"Job with name '{body['name']}' already exists")
|
| 985 |
+
|
| 986 |
+
_check_ssrf(body["url"])
|
| 987 |
+
|
| 988 |
+
job_id = str(uuid.uuid4())
|
| 989 |
+
now = datetime.now(UTC).isoformat()
|
| 990 |
+
|
| 991 |
+
trigger_config = dict(body.get("trigger", {}))
|
| 992 |
+
trigger_type = trigger_config.pop("type", "cron")
|
| 993 |
+
|
| 994 |
+
retry = body.get("retry", {})
|
| 995 |
+
|
| 996 |
+
job_data: dict[str, Any] = {
|
| 997 |
+
"id": job_id,
|
| 998 |
+
"name": body["name"],
|
| 999 |
+
"description": body.get("description"),
|
| 1000 |
+
"url": body["url"],
|
| 1001 |
+
"method": body.get("method", "GET"),
|
| 1002 |
+
"headers": body.get("headers", {}),
|
| 1003 |
+
"query_params": body.get("query_params", {}),
|
| 1004 |
+
"body": body.get("body"),
|
| 1005 |
+
"body_type": body.get("body_type", "json"),
|
| 1006 |
+
"auth_type": body.get("auth", {}).get("type", "none"),
|
| 1007 |
+
"auth_config": dict(body.get("auth", {})),
|
| 1008 |
+
"trigger_type": trigger_type,
|
| 1009 |
+
"trigger_config": trigger_config,
|
| 1010 |
+
"timezone": body.get("timezone", "UTC"),
|
| 1011 |
+
"timeout_seconds": body.get("timeout", 30.0),
|
| 1012 |
+
"max_retries": retry.get("max_retries", 3),
|
| 1013 |
+
"retry_delay_seconds": retry.get("retry_delay", 1.0),
|
| 1014 |
+
"retry_backoff": retry.get("retry_backoff", 2.0),
|
| 1015 |
+
"retry_max_delay_seconds": retry.get("retry_max_delay", 60.0),
|
| 1016 |
+
"retry_on_status": retry.get("retry_on_status", [429, 500, 502, 503, 504]),
|
| 1017 |
+
"retry_on_timeout": retry.get("retry_on_timeout", True),
|
| 1018 |
+
"retry_on_failure": retry.get("retry_on_failure", True),
|
| 1019 |
+
"retry_jitter": retry.get("retry_jitter", False),
|
| 1020 |
+
"max_instances": body.get("max_instances", 1),
|
| 1021 |
+
"coalesce": body.get("coalesce", True),
|
| 1022 |
+
"misfire_grace_time": body.get("misfire_grace_time"),
|
| 1023 |
+
"jitter_seconds": trigger_config.get("jitter"),
|
| 1024 |
+
"start_date": trigger_config.get("start_date"),
|
| 1025 |
+
"end_date": trigger_config.get("end_date"),
|
| 1026 |
+
"status": JobStatus.ACTIVE.value,
|
| 1027 |
+
"tags": body.get("tags", []),
|
| 1028 |
+
"metadata": body.get("metadata", {}),
|
| 1029 |
+
"created_at": now,
|
| 1030 |
+
"updated_at": now,
|
| 1031 |
+
"run_count": 0,
|
| 1032 |
+
"failure_count": 0,
|
| 1033 |
+
"success_count": 0,
|
| 1034 |
+
}
|
| 1035 |
+
|
| 1036 |
+
created = await repo.create_job(job_data)
|
| 1037 |
+
self._schedule_job(job_data)
|
| 1038 |
+
|
| 1039 |
+
next_run = self.get_next_run_time(job_id)
|
| 1040 |
+
if next_run:
|
| 1041 |
+
await repo.update_job(job_id, {"next_run_at": next_run.isoformat()})
|
| 1042 |
+
created["next_run_at"] = next_run.isoformat()
|
| 1043 |
+
|
| 1044 |
+
logger.info("Job created: %s (%s)", body["name"], job_id)
|
| 1045 |
+
return await repo.get_job_by_id(job_id) or created
|
| 1046 |
+
|
| 1047 |
+
async def update_job(self, job_id: str, body: dict[str, Any]) -> dict[str, Any]:
|
| 1048 |
+
client = get_supabase_client()
|
| 1049 |
+
if client is None:
|
| 1050 |
+
raise RuntimeError("Supabase not available")
|
| 1051 |
+
|
| 1052 |
+
repo = SchedulerRepository(client)
|
| 1053 |
+
job = await repo.get_job_by_id(job_id)
|
| 1054 |
+
if not job:
|
| 1055 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1056 |
+
if job.get("status") == JobStatus.DELETED.value:
|
| 1057 |
+
raise ValueError("Cannot update a deleted job")
|
| 1058 |
+
|
| 1059 |
+
update_data: dict[str, Any] = {}
|
| 1060 |
+
|
| 1061 |
+
field_map = {
|
| 1062 |
+
"description": "description",
|
| 1063 |
+
"url": "url",
|
| 1064 |
+
"method": "method",
|
| 1065 |
+
"headers": "headers",
|
| 1066 |
+
"query_params": "query_params",
|
| 1067 |
+
"body": "body",
|
| 1068 |
+
"body_type": "body_type",
|
| 1069 |
+
"timezone": "timezone",
|
| 1070 |
+
"timeout": "timeout_seconds",
|
| 1071 |
+
"max_instances": "max_instances",
|
| 1072 |
+
"coalesce": "coalesce",
|
| 1073 |
+
"misfire_grace_time": "misfire_grace_time",
|
| 1074 |
+
"tags": "tags",
|
| 1075 |
+
"metadata": "metadata",
|
| 1076 |
+
}
|
| 1077 |
+
|
| 1078 |
+
for req_key, db_key in field_map.items():
|
| 1079 |
+
if req_key in body and body[req_key] is not None:
|
| 1080 |
+
update_data[db_key] = body[req_key]
|
| 1081 |
+
|
| 1082 |
+
if "url" in body and body["url"] is not None:
|
| 1083 |
+
_check_ssrf(body["url"])
|
| 1084 |
+
|
| 1085 |
+
if "auth" in body and body["auth"] is not None:
|
| 1086 |
+
update_data["auth_type"] = body["auth"].get("type", "none")
|
| 1087 |
+
update_data["auth_config"] = dict(body["auth"])
|
| 1088 |
+
|
| 1089 |
+
if "retry" in body and body["retry"] is not None:
|
| 1090 |
+
r = body["retry"]
|
| 1091 |
+
update_data["max_retries"] = r.get("max_retries", 3)
|
| 1092 |
+
update_data["retry_delay_seconds"] = r.get("retry_delay", 1.0)
|
| 1093 |
+
update_data["retry_backoff"] = r.get("retry_backoff", 2.0)
|
| 1094 |
+
update_data["retry_max_delay_seconds"] = r.get("retry_max_delay", 60.0)
|
| 1095 |
+
update_data["retry_on_status"] = r.get("retry_on_status", [429, 500, 502, 503, 504])
|
| 1096 |
+
update_data["retry_on_timeout"] = r.get("retry_on_timeout", True)
|
| 1097 |
+
update_data["retry_on_failure"] = r.get("retry_on_failure", True)
|
| 1098 |
+
update_data["retry_jitter"] = r.get("retry_jitter", False)
|
| 1099 |
+
|
| 1100 |
+
if "trigger" in body and body["trigger"] is not None:
|
| 1101 |
+
tc = dict(body["trigger"])
|
| 1102 |
+
update_data["trigger_type"] = tc.pop("type", job.get("trigger_type", "cron"))
|
| 1103 |
+
update_data["trigger_config"] = tc
|
| 1104 |
+
update_data["jitter_seconds"] = tc.get("jitter")
|
| 1105 |
+
update_data["start_date"] = tc.get("start_date")
|
| 1106 |
+
update_data["end_date"] = tc.get("end_date")
|
| 1107 |
+
|
| 1108 |
+
await repo.update_job(job_id, update_data)
|
| 1109 |
+
updated = await repo.get_job_by_id(job_id)
|
| 1110 |
+
if updated is None:
|
| 1111 |
+
raise KeyError(f"Job '{job_id}' not found after update")
|
| 1112 |
+
|
| 1113 |
+
self._reschedule_job(updated)
|
| 1114 |
+
|
| 1115 |
+
logger.info("Job updated: %s", job_id)
|
| 1116 |
+
return updated
|
| 1117 |
+
|
| 1118 |
+
async def delete_job(self, job_id: str) -> None:
|
| 1119 |
+
client = get_supabase_client()
|
| 1120 |
+
if client is None:
|
| 1121 |
+
raise RuntimeError("Supabase not available")
|
| 1122 |
+
|
| 1123 |
+
repo = SchedulerRepository(client)
|
| 1124 |
+
job = await repo.get_job_by_id(job_id)
|
| 1125 |
+
if not job:
|
| 1126 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1127 |
+
|
| 1128 |
+
self._remove_aps_job(job_id)
|
| 1129 |
+
await repo.update_job(job_id, {"status": JobStatus.DELETED.value})
|
| 1130 |
+
logger.info("Job deleted: %s", job_id)
|
| 1131 |
+
|
| 1132 |
+
async def hard_delete_job(self, job_id: str) -> None:
|
| 1133 |
+
client = get_supabase_client()
|
| 1134 |
+
if client is None:
|
| 1135 |
+
raise RuntimeError("Supabase not available")
|
| 1136 |
+
|
| 1137 |
+
repo = SchedulerRepository(client)
|
| 1138 |
+
job = await repo.get_job_by_id(job_id)
|
| 1139 |
+
if not job:
|
| 1140 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1141 |
+
|
| 1142 |
+
self._remove_aps_job(job_id)
|
| 1143 |
+
await repo.delete_job(job_id)
|
| 1144 |
+
logger.info("Job hard deleted: %s", job_id)
|
| 1145 |
+
|
| 1146 |
+
async def pause_job(self, job_id: str) -> dict[str, Any]:
|
| 1147 |
+
client = get_supabase_client()
|
| 1148 |
+
if client is None:
|
| 1149 |
+
raise RuntimeError("Supabase not available")
|
| 1150 |
+
|
| 1151 |
+
repo = SchedulerRepository(client)
|
| 1152 |
+
job = await repo.get_job_by_id(job_id)
|
| 1153 |
+
if not job:
|
| 1154 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1155 |
+
if job.get("status") != JobStatus.ACTIVE.value:
|
| 1156 |
+
raise ValueError(f"Job is not active (current status: {job.get('status')})")
|
| 1157 |
+
|
| 1158 |
+
self.pause_aps_job(job_id)
|
| 1159 |
+
await repo.update_job(job_id, {"status": JobStatus.PAUSED.value})
|
| 1160 |
+
updated = await repo.get_job_by_id(job_id)
|
| 1161 |
+
logger.info("Job paused: %s", job_id)
|
| 1162 |
+
return updated or job
|
| 1163 |
+
|
| 1164 |
+
async def resume_job(self, job_id: str) -> dict[str, Any]:
|
| 1165 |
+
client = get_supabase_client()
|
| 1166 |
+
if client is None:
|
| 1167 |
+
raise RuntimeError("Supabase not available")
|
| 1168 |
+
|
| 1169 |
+
repo = SchedulerRepository(client)
|
| 1170 |
+
job = await repo.get_job_by_id(job_id)
|
| 1171 |
+
if not job:
|
| 1172 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1173 |
+
if job.get("status") != JobStatus.PAUSED.value:
|
| 1174 |
+
raise ValueError(f"Job is not paused (current status: {job.get('status')})")
|
| 1175 |
+
|
| 1176 |
+
self.resume_aps_job(job_id)
|
| 1177 |
+
await repo.update_job(job_id, {"status": JobStatus.ACTIVE.value})
|
| 1178 |
+
updated = await repo.get_job_by_id(job_id)
|
| 1179 |
+
logger.info("Job resumed: %s", job_id)
|
| 1180 |
+
return updated or job
|
| 1181 |
+
|
| 1182 |
+
async def run_job_now(self, job_id: str) -> None:
|
| 1183 |
+
client = get_supabase_client()
|
| 1184 |
+
if client is None:
|
| 1185 |
+
raise RuntimeError("Supabase not available")
|
| 1186 |
+
|
| 1187 |
+
repo = SchedulerRepository(client)
|
| 1188 |
+
job = await repo.get_job_by_id(job_id)
|
| 1189 |
+
if not job:
|
| 1190 |
+
raise KeyError(f"Job '{job_id}' not found")
|
| 1191 |
+
if job.get("status") == JobStatus.DELETED.value:
|
| 1192 |
+
raise ValueError("Cannot run a deleted job")
|
| 1193 |
+
|
| 1194 |
+
asyncio.create_task(self._execute_job(job_id, TriggerReason.MANUAL))
|
| 1195 |
+
logger.info("Job triggered manually: %s", job_id)
|
| 1196 |
+
|
| 1197 |
+
async def list_jobs(
|
| 1198 |
+
self,
|
| 1199 |
+
status: str | None = None,
|
| 1200 |
+
tags: list[str] | None = None,
|
| 1201 |
+
page: int = 1,
|
| 1202 |
+
page_size: int = 20,
|
| 1203 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 1204 |
+
client = get_supabase_client()
|
| 1205 |
+
if client is None:
|
| 1206 |
+
return [], 0
|
| 1207 |
+
repo = SchedulerRepository(client)
|
| 1208 |
+
return await repo.list_jobs(status=status, tags=tags, page=page, page_size=page_size)
|
| 1209 |
+
|
| 1210 |
+
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
| 1211 |
+
client = get_supabase_client()
|
| 1212 |
+
if client is None:
|
| 1213 |
+
return None
|
| 1214 |
+
repo = SchedulerRepository(client)
|
| 1215 |
+
return await repo.get_job_by_id(job_id)
|
| 1216 |
+
|
| 1217 |
+
async def get_job_history(
|
| 1218 |
+
self,
|
| 1219 |
+
job_id: str,
|
| 1220 |
+
page: int = 1,
|
| 1221 |
+
page_size: int = 20,
|
| 1222 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 1223 |
+
client = get_supabase_client()
|
| 1224 |
+
if client is None:
|
| 1225 |
+
return [], 0
|
| 1226 |
+
repo = SchedulerRepository(client)
|
| 1227 |
+
return await repo.get_history(job_id=job_id, page=page, page_size=page_size)
|
| 1228 |
+
|
| 1229 |
+
async def get_execution_history(
|
| 1230 |
+
self,
|
| 1231 |
+
page: int = 1,
|
| 1232 |
+
page_size: int = 50,
|
| 1233 |
+
status: str | None = None,
|
| 1234 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 1235 |
+
client = get_supabase_client()
|
| 1236 |
+
if client is None:
|
| 1237 |
+
return [], 0
|
| 1238 |
+
repo = SchedulerRepository(client)
|
| 1239 |
+
return await repo.get_history_all(page=page, page_size=page_size, status_filter=status)
|
| 1240 |
+
|
| 1241 |
+
async def get_metrics(self) -> dict[str, Any]:
|
| 1242 |
+
client = get_supabase_client()
|
| 1243 |
+
if client is None:
|
| 1244 |
+
return {"error": "Supabase not available"}
|
| 1245 |
+
|
| 1246 |
+
repo = SchedulerRepository(client)
|
| 1247 |
+
total_jobs = await repo.count_jobs()
|
| 1248 |
+
active_jobs = await repo.count_jobs(status=JobStatus.ACTIVE.value)
|
| 1249 |
+
paused_jobs = await repo.count_jobs(status=JobStatus.PAUSED.value)
|
| 1250 |
+
total_executions = await repo.count_executions()
|
| 1251 |
+
successful_executions = await repo.count_executions(status=ExecutionStatus.SUCCESS.value)
|
| 1252 |
+
|
| 1253 |
+
scheduler_status = self.get_scheduler_status()
|
| 1254 |
+
|
| 1255 |
+
return {
|
| 1256 |
+
"jobs": {
|
| 1257 |
+
"total": total_jobs,
|
| 1258 |
+
"active": active_jobs,
|
| 1259 |
+
"paused": paused_jobs,
|
| 1260 |
+
},
|
| 1261 |
+
"executions": {
|
| 1262 |
+
"total": total_executions,
|
| 1263 |
+
"success": successful_executions,
|
| 1264 |
+
"failure": total_executions - successful_executions,
|
| 1265 |
+
"success_rate": round(successful_executions / total_executions * 100, 2) if total_executions else 0,
|
| 1266 |
+
},
|
| 1267 |
+
"scheduler": scheduler_status,
|
| 1268 |
+
}
|
| 1269 |
+
|
| 1270 |
+
|
| 1271 |
+
# ---------------------------------------------------------------------------
|
| 1272 |
+
# Validation helpers
|
| 1273 |
+
# ---------------------------------------------------------------------------
|
| 1274 |
+
|
| 1275 |
+
def validate_cron_expression(expression: str) -> None:
|
| 1276 |
+
try:
|
| 1277 |
+
croniter(expression)
|
| 1278 |
+
except (ValueError, KeyError) as exc:
|
| 1279 |
+
raise ValueError(f"Invalid cron expression '{expression}': {exc}") from exc
|
| 1280 |
+
|
| 1281 |
+
|
| 1282 |
+
def validate_timezone(tz: str) -> None:
|
| 1283 |
+
try:
|
| 1284 |
+
import pytz
|
| 1285 |
+
pytz.timezone(tz)
|
| 1286 |
+
except Exception as exc:
|
| 1287 |
+
raise ValueError(f"Unknown timezone: '{tz}'") from exc
|
| 1288 |
+
|
| 1289 |
+
|
| 1290 |
+
def validate_url(url: str) -> str:
|
| 1291 |
+
if not url.startswith(("http://", "https://")):
|
| 1292 |
+
raise ValueError("Only http/https URLs are supported")
|
| 1293 |
+
return url
|
| 1294 |
+
|
| 1295 |
+
|
| 1296 |
+
def job_to_response(job: dict[str, Any]) -> dict[str, Any]:
|
| 1297 |
+
return {
|
| 1298 |
+
"id": job.get("id"),
|
| 1299 |
+
"name": job.get("name"),
|
| 1300 |
+
"description": job.get("description"),
|
| 1301 |
+
"url": job.get("url"),
|
| 1302 |
+
"method": job.get("method", "GET"),
|
| 1303 |
+
"headers": job.get("headers", {}),
|
| 1304 |
+
"query_params": job.get("query_params", {}),
|
| 1305 |
+
"body": job.get("body"),
|
| 1306 |
+
"body_type": job.get("body_type", "json"),
|
| 1307 |
+
"auth_type": job.get("auth_type", "none"),
|
| 1308 |
+
"trigger_type": job.get("trigger_type"),
|
| 1309 |
+
"trigger_config": job.get("trigger_config", {}),
|
| 1310 |
+
"timezone": job.get("timezone", "UTC"),
|
| 1311 |
+
"timeout": job.get("timeout_seconds", 30.0),
|
| 1312 |
+
"max_retries": job.get("max_retries", 3),
|
| 1313 |
+
"retry_delay": job.get("retry_delay_seconds", 1.0),
|
| 1314 |
+
"retry_backoff": job.get("retry_backoff", 2.0),
|
| 1315 |
+
"retry_max_delay": job.get("retry_max_delay_seconds", 60.0),
|
| 1316 |
+
"retry_on_status": job.get("retry_on_status", [429, 500, 502, 503, 504]),
|
| 1317 |
+
"retry_on_timeout": job.get("retry_on_timeout", True),
|
| 1318 |
+
"retry_on_failure": job.get("retry_on_failure", True),
|
| 1319 |
+
"retry_jitter": job.get("retry_jitter", False),
|
| 1320 |
+
"max_instances": job.get("max_instances", 1),
|
| 1321 |
+
"coalesce": job.get("coalesce", True),
|
| 1322 |
+
"misfire_grace_time": job.get("misfire_grace_time"),
|
| 1323 |
+
"jitter": job.get("jitter_seconds"),
|
| 1324 |
+
"start_date": job.get("start_date"),
|
| 1325 |
+
"end_date": job.get("end_date"),
|
| 1326 |
+
"status": job.get("status", "active"),
|
| 1327 |
+
"tags": job.get("tags", []),
|
| 1328 |
+
"metadata": job.get("metadata", {}),
|
| 1329 |
+
"created_at": job.get("created_at"),
|
| 1330 |
+
"updated_at": job.get("updated_at"),
|
| 1331 |
+
"last_run_at": job.get("last_run_at"),
|
| 1332 |
+
"next_run_at": job.get("next_run_at"),
|
| 1333 |
+
"run_count": job.get("run_count", 0),
|
| 1334 |
+
"failure_count": job.get("failure_count", 0),
|
| 1335 |
+
"success_count": job.get("success_count", 0),
|
| 1336 |
+
}
|
| 1337 |
+
|
| 1338 |
+
|
| 1339 |
+
def history_to_response(h: dict[str, Any]) -> dict[str, Any]:
|
| 1340 |
+
return {
|
| 1341 |
+
"id": h.get("id"),
|
| 1342 |
+
"job_id": h.get("job_id"),
|
| 1343 |
+
"job_name": h.get("job_name"),
|
| 1344 |
+
"started_at": h.get("started_at"),
|
| 1345 |
+
"ended_at": h.get("ended_at"),
|
| 1346 |
+
"duration_ms": h.get("duration_ms"),
|
| 1347 |
+
"status": h.get("status"),
|
| 1348 |
+
"http_status_code": h.get("http_status_code"),
|
| 1349 |
+
"response_size_bytes": h.get("response_size_bytes"),
|
| 1350 |
+
"retry_count": h.get("retry_count", 0),
|
| 1351 |
+
"trigger_reason": h.get("trigger_reason", "scheduled"),
|
| 1352 |
+
"error_message": h.get("error_message"),
|
| 1353 |
+
"exception_type": h.get("exception_type"),
|
| 1354 |
+
"request_url": h.get("request_url"),
|
| 1355 |
+
"request_method": h.get("request_method"),
|
| 1356 |
+
"response_preview": h.get("response_preview"),
|
| 1357 |
+
"created_at": h.get("created_at"),
|
| 1358 |
+
}
|
pyproject.toml
CHANGED
|
@@ -33,6 +33,10 @@ dependencies = [
|
|
| 33 |
"phonenumbers>=8.13.0",
|
| 34 |
"jsonschema>=4.21.0",
|
| 35 |
"supabase>=2.0.0",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
]
|
| 37 |
|
| 38 |
[project.optional-dependencies]
|
|
|
|
| 33 |
"phonenumbers>=8.13.0",
|
| 34 |
"jsonschema>=4.21.0",
|
| 35 |
"supabase>=2.0.0",
|
| 36 |
+
"apscheduler>=3.10.4",
|
| 37 |
+
"croniter>=2.0.0",
|
| 38 |
+
"pytz>=2024.1",
|
| 39 |
+
"redis>=5.0.0",
|
| 40 |
]
|
| 41 |
|
| 42 |
[project.optional-dependencies]
|
requirements.txt
CHANGED
|
@@ -67,3 +67,8 @@ jsonpath-ng>=1.7.0
|
|
| 67 |
|
| 68 |
# QR code generation (testing)
|
| 69 |
qrcode[pil]>=8.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
# QR code generation (testing)
|
| 69 |
qrcode[pil]>=8.0
|
| 70 |
+
|
| 71 |
+
# Scheduler
|
| 72 |
+
apscheduler>=3.10.4
|
| 73 |
+
croniter>=2.0.0
|
| 74 |
+
pytz>=2024.1
|