refactored background jobs

app/main.py

@@ -46,7 +46,20 @@ async def lifespan(app: FastAPI):
     
     app.state.instance_id = f"engine-{uuid.uuid4().hex[:8]}"
     logger.info(f"Instance ID: {app.state.instance_id}")
+    logger.info("🚀 STARTUP SEQUENCE")
     
+    # ✅ CRITICAL: Set persistent cache dir (survives restarts)
+    os.makedirs("/data/hf_cache", exist_ok=True)
+    os.environ["HF_HOME"] = "/data/hf_cache"
+    os.environ["TRANSFORMERS_CACHE"] = "/data/hf_cache"
+    os.environ["HF_HUB_CACHE"] = "/data/hf_cache"
+    
+    # Set Hugging Face cache symlink (if needed)
+    cache_dir = pathlib.Path("/data/hf_cache")
+    home_cache = pathlib.Path.home() / ".cache" / "huggingface"
+    if not home_cache.exists():
+        home_cache.parent.mkdir(parents=True, exist_ok=True)
+        home_cache.symlink_to(cache_dir)
     # Validate service health on boot
     try:
         services = check_all_services()
@@ -93,6 +106,10 @@ async def lifespan(app: FastAPI):
     # Start background KPI scheduler
     logger.info("⏰ Starting KPI refresh scheduler...")
     asyncio.create_task(continuous_kpi_refresh(), name="kpi_scheduler")
+    # Now load LLM service - it will use persistent cache
+    from app.service.llm_service import LocalLLMService
+    logger.info("🤖 LLM service initialized (will use persistent cache)")
+    
     yield
     
     # ─── Shutdown ──────────────────────────────────────────────────────────────

app/qstash_client.py

@@ -1 +1,69 @@
-qstash_client.py
+# app/qstash_client.py
+import os
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# ✅ Module-level singleton (survives imports)
+_qstash_client: Optional["Client"] = None
+
+def init_qstash_client() -> bool:
+    """
+    EXPLICITLY initialize QStash client on startup.
+    Call this AFTER HF secrets are loaded.
+    
+    Returns:
+        bool: True if successful, False if QStash not configured
+    """
+    global _qstash_client
+    
+    if _qstash_client is not None:
+        logger.info("✅ QStash client already initialized")
+        return True
+    
+    token = os.getenv("QSTASH_TOKEN")
+    if not token:
+        logger.warning("⚠️ QSTASH_TOKEN not found - QStash features disabled")
+        return False
+    
+    try:
+        from upstash_qstash import Client
+        
+        qstash_url = os.getenv("QSTASH_URL")
+        if qstash_url:
+            _qstash_client = Client(token=token, url=qstash_url)
+            logger.info(f"✅ QStash client initialized (custom URL: {qstash_url})")
+        else:
+            _qstash_client = Client(token=token)
+            logger.info("✅ QStash client initialized")
+        
+        return True
+        
+    except ImportError:
+        logger.error("❌ upstash_qstash not installed. pip install upstash-qstash")
+        return False
+    except Exception as e:
+        logger.error(f"❌ QStash initialization failed: {e}")
+        return False
+
+def get_qstash_client() -> Optional["Client"]:
+    """
+    Get the singleton QStash client.
+    
+    Returns:
+        Client instance or None if not initialized
+        
+    Raises:
+        RuntimeError: If called before init_qstash_client()
+    """
+    if _qstash_client is None:
+        raise RuntimeError(
+            "QStash client not initialized. "
+            "Call init_qstash_client() in startup sequence first."
+        )
+    return _qstash_client
+
+def is_qstash_available() -> bool:
+    """Check if QStash is configured and ready"""
+    return _qstash_client is not None

app/routers/datasources.py

@@ -32,7 +32,6 @@ async def create_source_json(
     orgId: str = Query(...),      # ✅ From Vercel
     sourceId: str = Query(...),   # ✅ From Vercel
     type: str = Query(...),       # ✅ From Vercel
-    background_tasks: BackgroundTasks,
     current_user: dict = Depends(get_current_user),  # ✅ For auth
     _: str = Depends(verify_api_key),
 ):

app/service/llm_service.py

@@ -20,11 +20,15 @@ class LocalLLMService:
         self._load_error = None
         self._lock = Lock()
         
+        # ✅ Use persistent cache
+        cache_dir = "/data/hf_cache"
+        os.makedirs(cache_dir, exist_ok=True)
+        
         logger.info("🚀 Starting background LLM load...")
         Thread(target=self._load_model_background, daemon=True).start()
     
     def _load_model_background(self):
-        """Load model in background thread"""
+        """Load model in background thread with persistent cache"""
         with self._lock:
             if self._is_loading or self._is_loaded:
                 return
@@ -33,12 +37,17 @@ class LocalLLMService:
         try:
             logger.info(f"🤖 [BACKGROUND] Loading LLM: {self.model_id}...")
             
+            # ✅ Use persistent cache directory
+            cache_dir = "/data/hf_cache"
+            
             # Phi-3 tokenizer
             self._tokenizer = AutoTokenizer.from_pretrained(
                 self.model_id,
                 token=HF_API_TOKEN,
-                trust_remote_code=True
+                trust_remote_code=True,
+                cache_dir=cache_dir  # ✅ Persistent cache
             )
+            # .
             self._tokenizer.pad_token = self._tokenizer.eos_token
             
             # Phi-3 model - OPTIMIZED for speed