Upload 295 files

ai_models.py

@@ -15,6 +15,19 @@ try:
 except ImportError:
     TRANSFORMERS_AVAILABLE = False
 
+try:
+    from huggingface_hub.errors import RepositoryNotFoundError
+    HF_HUB_AVAILABLE = True
+except ImportError:
+    HF_HUB_AVAILABLE = False
+    RepositoryNotFoundError = Exception
+
+try:
+    import requests
+    REQUESTS_AVAILABLE = True
+except ImportError:
+    REQUESTS_AVAILABLE = False
+
 logger = logging.getLogger(__name__)
 settings = get_settings()
 
@@ -31,17 +44,26 @@ if HF_MODE == "auth" and not HF_TOKEN_ENV:
     HF_MODE = "off"
     logger.warning("HF_MODE='auth' but no HF_TOKEN found, resetting to 'off'")
 
-# Extended Model Catalog
+# Extended Model Catalog - Updated with valid public models
+# Primary models first, fallbacks follow
 CRYPTO_SENTIMENT_MODELS = [
-    "ElKulako/cryptobert", "kk08/CryptoBERT",
-    "burakutf/finetuned-finbert-crypto", "mathugo/crypto_news_bert"
+    "cardiffnlp/twitter-roberta-base-sentiment-latest",  # Primary: reliable public model
+    "kk08/CryptoBERT",  # Fallback 1
+    "burakutf/finetuned-finbert-crypto",  # Fallback 2
+    "mathugo/crypto_news_bert"  # Fallback 3
 ]
 SOCIAL_SENTIMENT_MODELS = [
-    "svalabs/twitter-xlm-roberta-bitcoin-sentiment",
-    "mayurjadhav/crypto-sentiment-model"
+    "cardiffnlp/twitter-roberta-base-sentiment-latest",  # Primary: reliable public model
+    "mayurjadhav/crypto-sentiment-model"  # Fallback
+]
+FINANCIAL_SENTIMENT_MODELS = [
+    "cardiffnlp/twitter-roberta-base-sentiment-latest",  # Primary: reliable public model
+    "ProsusAI/finbert"  # Fallback (may require auth)
+]
+NEWS_SENTIMENT_MODELS = [
+    "cardiffnlp/twitter-roberta-base-sentiment-latest",  # Primary: reliable public model
+    "mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"  # Fallback
 ]
-FINANCIAL_SENTIMENT_MODELS = ["ProsusAI/finbert", "cardiffnlp/twitter-roberta-base-sentiment"]
-NEWS_SENTIMENT_MODELS = ["mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"]
 DECISION_MODELS = ["agarkovv/CryptoTrader-LM"]
 
 @dataclass(frozen=True)
@@ -96,8 +118,29 @@ class ModelRegistry:
         self._pipelines = {}
         self._lock = threading.Lock()
         self._initialized = False
+        self._failed_models = {}  # Track failed models with reasons
+
+    def _should_use_token(self, spec: PipelineSpec) -> Optional[str]:
+        """Determine if and which token to use for model loading"""
+        if HF_MODE == "off":
+            return None
+        
+        # In public mode, don't use token even if requires_auth
+        if HF_MODE == "public":
+            return None
+        
+        # In auth mode, use token if available
+        if HF_MODE == "auth":
+            if spec.requires_auth and HF_TOKEN_ENV:
+                return HF_TOKEN_ENV
+            elif spec.requires_auth and not HF_TOKEN_ENV:
+                logger.warning(f"Model {spec.model_id} requires auth but no token available")
+                return None
+        
+        return None
 
     def get_pipeline(self, key: str):
+        """Get pipeline for a model key, with robust error handling"""
         if HF_MODE == "off":
             raise ModelNotAvailable("HF_MODE=off")
         if not TRANSFORMERS_AVAILABLE:
@@ -106,103 +149,285 @@ class ModelRegistry:
             raise ModelNotAvailable(f"Unknown key: {key}")
         
         spec = MODEL_SPECS[key]
+        
+        # Return cached pipeline if available
         if key in self._pipelines:
             return self._pipelines[key]
         
+        # Check if this model already failed
+        if key in self._failed_models:
+            raise ModelNotAvailable(f"Model failed previously: {self._failed_models[key]}")
+        
         with self._lock:
+            # Double-check after acquiring lock
             if key in self._pipelines:
                 return self._pipelines[key]
+            if key in self._failed_models:
+                raise ModelNotAvailable(f"Model failed previously: {self._failed_models[key]}")
+            
+            # Determine token usage
+            auth_token = self._should_use_token(spec)
+            
+            logger.info(f"Loading model: {spec.model_id} (mode={HF_MODE}, auth={'yes' if auth_token else 'no'})")
             
-            auth = HF_TOKEN_ENV if (HF_MODE == "auth" and spec.requires_auth) else (HF_TOKEN_ENV if spec.requires_auth else None)
-            logger.info(f"Loading model: {spec.model_id}")
             try:
-                self._pipelines[key] = pipeline(spec.task, model=spec.model_id, tokenizer=spec.model_id, use_auth_token=auth)
+                # Use token parameter instead of deprecated use_auth_token
+                pipeline_kwargs = {
+                    "task": spec.task,
+                    "model": spec.model_id,
+                }
+                
+                # Only add token if we have one and it's needed
+                if auth_token:
+                    pipeline_kwargs["token"] = auth_token
+                else:
+                    # Explicitly set to None to avoid using expired tokens
+                    pipeline_kwargs["token"] = None
+                
+                self._pipelines[key] = pipeline(**pipeline_kwargs)
+                logger.info(f"Successfully loaded model: {spec.model_id}")
+                return self._pipelines[key]
+                
+            except RepositoryNotFoundError as e:
+                error_msg = f"Repository not found: {spec.model_id}"
+                logger.warning(f"{error_msg} - {str(e)}")
+                self._failed_models[key] = error_msg
+                raise ModelNotAvailable(error_msg) from e
+                
             except Exception as e:
-                logger.exception(f"Failed to load {spec.model_id}")
-                raise ModelNotAvailable(str(e)) from e
+                error_type = type(e).__name__
+                error_msg = f"{error_type}: {str(e)[:100]}"
+                
+                # Check for HTTP errors (401, 403, 404)
+                if REQUESTS_AVAILABLE and isinstance(e, requests.exceptions.HTTPError):
+                    status_code = getattr(e.response, 'status_code', None)
+                    if status_code == 401:
+                        error_msg = f"Authentication failed (401) for {spec.model_id}"
+                    elif status_code == 403:
+                        error_msg = f"Access forbidden (403) for {spec.model_id}"
+                    elif status_code == 404:
+                        error_msg = f"Model not found (404): {spec.model_id}"
+                
+                # Check for OSError from transformers
+                if isinstance(e, OSError) and "not a valid model identifier" in str(e):
+                    error_msg = f"Invalid model identifier: {spec.model_id}"
+                
+                logger.warning(f"Failed to load {spec.model_id}: {error_msg}")
+                self._failed_models[key] = error_msg
+                raise ModelNotAvailable(error_msg) from e
         
         return self._pipelines[key]
 
     def initialize_models(self):
+        """Initialize models with fallback logic - tries primary models first"""
         if self._initialized:
-            return {"status": "already_initialized", "mode": HF_MODE, "models_loaded": len(self._pipelines)}
+            return {
+                "status": "already_initialized",
+                "mode": HF_MODE,
+                "models_loaded": len(self._pipelines),
+                "failed_count": len(self._failed_models)
+            }
+        
         if HF_MODE == "off":
-            return {"status": "disabled", "mode": HF_MODE, "models_loaded": 0, "error": "HF_MODE=off"}
+            return {
+                "status": "disabled",
+                "mode": HF_MODE,
+                "models_loaded": 0,
+                "error": "HF_MODE=off"
+            }
+        
         if not TRANSFORMERS_AVAILABLE:
-            return {"status": "transformers_not_available", "mode": HF_MODE, "models_loaded": 0}
+            return {
+                "status": "transformers_not_available",
+                "mode": HF_MODE,
+                "models_loaded": 0,
+                "error": "transformers library not installed"
+            }
         
         loaded, failed = [], []
-        for key in ["crypto_sent_0", "financial_sent_0"]:
-            try:
-                self.get_pipeline(key)
-                loaded.append(key)
-            except Exception as e:
-                failed.append((key, str(e)))
+        
+        # Try to load at least one model from each category with fallback
+        categories_to_try = {
+            "crypto": ["crypto_sent_0", "crypto_sent_1", "crypto_sent_2"],
+            "financial": ["financial_sent_0", "financial_sent_1"],
+            "social": ["social_sent_0", "social_sent_1"],
+            "news": ["news_sent_0", "news_sent_1"]
+        }
+        
+        for category, keys in categories_to_try.items():
+            category_loaded = False
+            for key in keys:
+                if key not in MODEL_SPECS:
+                    continue
+                try:
+                    self.get_pipeline(key)
+                    loaded.append(key)
+                    category_loaded = True
+                    break  # Successfully loaded one from this category
+                except ModelNotAvailable as e:
+                    failed.append((key, str(e)[:100]))  # Truncate long errors
+                except Exception as e:
+                    failed.append((key, f"{type(e).__name__}: {str(e)[:100]}"))
+        
+        # Determine status
+        if len(loaded) > 0:
+            status = "ok" if len(loaded) >= 2 else "partial"
+        else:
+            status = "failed"
         
         self._initialized = True
-        return {"status": "ok" if loaded else "partial", "mode": HF_MODE, "models_loaded": len(loaded), "loaded": loaded, "failed": failed}
+        
+        return {
+            "status": status,
+            "mode": HF_MODE,
+            "models_loaded": len(loaded),
+            "models_failed": len(failed),
+            "loaded": loaded[:10],  # Limit to first 10 for brevity
+            "failed": failed[:10],  # Limit to first 10 for brevity
+            "failed_count": len(self._failed_models)
+        }
 
 _registry = ModelRegistry()
 
 def initialize_models(): return _registry.initialize_models()
 
 def ensemble_crypto_sentiment(text: str) -> Dict[str, Any]:
+    """Ensemble crypto sentiment with fallback model selection"""
     if not TRANSFORMERS_AVAILABLE:
-        return {"label": "neutral", "confidence": 0.0, "scores": {}, "model_count": 0, "error": "transformers N/A"}
+        return {"label": "neutral", "confidence": 0.0, "scores": {}, "model_count": 0, "error": "transformers N/A", "available": False}
+    
+    if HF_MODE == "off":
+        return {"label": "neutral", "confidence": 0.0, "scores": {}, "model_count": 0, "error": "HF_MODE=off", "available": False}
     
     results, labels_count, total_conf = {}, {"bullish": 0, "bearish": 0, "neutral": 0}, 0.0
     
-    for key in ["crypto_sent_0", "crypto_sent_1"]:
+    # Try models in order with fallback
+    candidate_keys = ["crypto_sent_0", "crypto_sent_1", "crypto_sent_2", "crypto_sent_3"]
+    
+    for key in candidate_keys:
+        if key not in MODEL_SPECS:
+            continue
         try:
             pipe = _registry.get_pipeline(key)
             res = pipe(text[:512])
-            if isinstance(res, list) and res: res = res[0]
+            if isinstance(res, list) and res: 
+                res = res[0]
             
             label = res.get("label", "NEUTRAL").upper()
             score = res.get("score", 0.5)
             
-            mapped = "bullish" if "POSITIVE" in label or "BULLISH" in label else ("bearish" if "NEGATIVE" in label or "BEARISH" in label else "neutral")
+            # Map labels to our standard format
+            mapped = "bullish" if "POSITIVE" in label or "BULLISH" in label or "LABEL_2" in label else (
+                "bearish" if "NEGATIVE" in label or "BEARISH" in label or "LABEL_0" in label else "neutral"
+            )
             
             spec = MODEL_SPECS[key]
             results[spec.model_id] = {"label": mapped, "score": score}
             labels_count[mapped] += 1
             total_conf += score
+            
+            # If we got at least one result, we can proceed
+            if len(results) >= 1:
+                break  # Got at least one working model
+                
+        except ModelNotAvailable:
+            continue  # Try next model
         except Exception as e:
-            logger.warning(f"Ensemble failed for {key}: {e}")
+            logger.warning(f"Ensemble failed for {key}: {str(e)[:100]}")
+            continue
     
     if not results:
-        return {"label": "neutral", "confidence": 0.0, "scores": {}, "model_count": 0}
+        return {
+            "label": "neutral",
+            "confidence": 0.0,
+            "scores": {},
+            "model_count": 0,
+            "available": False,
+            "error": "No models available"
+        }
     
     final = max(labels_count, key=labels_count.get)
     avg_conf = total_conf / len(results)
     
-    return {"label": final, "confidence": avg_conf, "scores": results, "model_count": len(results)}
+    return {
+        "label": final,
+        "confidence": avg_conf,
+        "scores": results,
+        "model_count": len(results),
+        "available": True
+    }
 
 def analyze_crypto_sentiment(text: str): return ensemble_crypto_sentiment(text)
 
 def analyze_financial_sentiment(text: str):
+    """Analyze financial sentiment with fallback"""
     if not TRANSFORMERS_AVAILABLE:
-        return {"label": "neutral", "score": 0.5, "error": "transformers N/A"}
-    try:
-        pipe = _registry.get_pipeline("financial_sent_0")
-        res = pipe(text[:512])
-        if isinstance(res, list) and res: res = res[0]
-        return {"label": res.get("label", "neutral").lower(), "score": res.get("score", 0.5)}
-    except Exception as e:
-        logger.error(f"Financial sentiment failed: {e}")
-        return {"label": "neutral", "score": 0.5, "error": str(e)}
+        return {"label": "neutral", "score": 0.5, "error": "transformers N/A", "available": False}
+    
+    if HF_MODE == "off":
+        return {"label": "neutral", "score": 0.5, "error": "HF_MODE=off", "available": False}
+    
+    # Try models in order
+    for key in ["financial_sent_0", "financial_sent_1"]:
+        if key not in MODEL_SPECS:
+            continue
+        try:
+            pipe = _registry.get_pipeline(key)
+            res = pipe(text[:512])
+            if isinstance(res, list) and res: 
+                res = res[0]
+            
+            label = res.get("label", "neutral").upper()
+            score = res.get("score", 0.5)
+            
+            # Map to standard format
+            mapped = "bullish" if "POSITIVE" in label or "LABEL_2" in label else (
+                "bearish" if "NEGATIVE" in label or "LABEL_0" in label else "neutral"
+            )
+            
+            return {"label": mapped, "score": score, "available": True, "model": MODEL_SPECS[key].model_id}
+        except ModelNotAvailable:
+            continue
+        except Exception as e:
+            logger.warning(f"Financial sentiment failed for {key}: {str(e)[:100]}")
+            continue
+    
+    return {"label": "neutral", "score": 0.5, "error": "No models available", "available": False}
 
 def analyze_social_sentiment(text: str):
+    """Analyze social sentiment with fallback"""
     if not TRANSFORMERS_AVAILABLE:
-        return {"label": "neutral", "score": 0.5, "error": "transformers N/A"}
-    try:
-        pipe = _registry.get_pipeline("social_sent_0")
-        res = pipe(text[:512])
-        if isinstance(res, list) and res: res = res[0]
-        return {"label": res.get("label", "neutral").lower(), "score": res.get("score", 0.5)}
-    except Exception as e:
-        logger.error(f"Social sentiment failed: {e}")
-        return {"label": "neutral", "score": 0.5, "error": str(e)}
+        return {"label": "neutral", "score": 0.5, "error": "transformers N/A", "available": False}
+    
+    if HF_MODE == "off":
+        return {"label": "neutral", "score": 0.5, "error": "HF_MODE=off", "available": False}
+    
+    # Try models in order
+    for key in ["social_sent_0", "social_sent_1"]:
+        if key not in MODEL_SPECS:
+            continue
+        try:
+            pipe = _registry.get_pipeline(key)
+            res = pipe(text[:512])
+            if isinstance(res, list) and res: 
+                res = res[0]
+            
+            label = res.get("label", "neutral").upper()
+            score = res.get("score", 0.5)
+            
+            # Map to standard format
+            mapped = "bullish" if "POSITIVE" in label or "LABEL_2" in label else (
+                "bearish" if "NEGATIVE" in label or "LABEL_0" in label else "neutral"
+            )
+            
+            return {"label": mapped, "score": score, "available": True, "model": MODEL_SPECS[key].model_id}
+        except ModelNotAvailable:
+            continue
+        except Exception as e:
+            logger.warning(f"Social sentiment failed for {key}: {str(e)[:100]}")
+            continue
+    
+    return {"label": "neutral", "score": 0.5, "error": "No models available", "available": False}
 
 def analyze_market_text(text: str): return ensemble_crypto_sentiment(text)
 
@@ -243,12 +468,24 @@ def get_model_info():
     }
 
 def registry_status():
-    return {
-        "ok": HF_MODE != "off",
+    """Get registry status with detailed information"""
+    status = {
+        "ok": HF_MODE != "off" and TRANSFORMERS_AVAILABLE and len(_registry._pipelines) > 0,
         "initialized": _registry._initialized,
         "pipelines_loaded": len(_registry._pipelines),
-        "available_models": list(MODEL_SPECS.keys()),
+        "pipelines_failed": len(_registry._failed_models),
+        "available_models": list(_registry._pipelines.keys()),
+        "failed_models": list(_registry._failed_models.keys())[:10],  # Limit for brevity
         "transformers_available": TRANSFORMERS_AVAILABLE,
         "hf_mode": HF_MODE,
-        "error": "HF_MODE=off" if HF_MODE == "off" else None
+        "total_specs": len(MODEL_SPECS)
     }
+    
+    if HF_MODE == "off":
+        status["error"] = "HF_MODE=off"
+    elif not TRANSFORMERS_AVAILABLE:
+        status["error"] = "transformers not installed"
+    elif len(_registry._pipelines) == 0 and _registry._initialized:
+        status["error"] = "No models loaded successfully"
+    
+    return status

api_server_extended.py

@@ -10,12 +10,15 @@ import sqlite3
 import httpx
 import json
 import subprocess
+import logging
 from pathlib import Path
 from typing import Optional, Dict, Any, List
 from datetime import datetime
 from contextlib import asynccontextmanager
 from collections import defaultdict
 
+logger = logging.getLogger(__name__)
+
 from fastapi import FastAPI, HTTPException, Response, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
@@ -1046,39 +1049,66 @@ async def analyze_sentiment(request: Dict[str, Any]):
             else:
                 result = analyze_market_text(text)
             
+            # Check if models are available
+            if not result.get("available", True):
+                return {
+                    "success": False,
+                    "available": False,
+                    "sentiment": "neutral",
+                    "confidence": 0.0,
+                    "mode": mode,
+                    "error": result.get("error", "Models not available"),
+                    "reason": "model_unavailable"
+                }
+            
             sentiment_label = result.get("label", "neutral")
             confidence = result.get("confidence", result.get("score", 0.5))
-            model_used = result.get("model_count", 0)
+            model_used = result.get("model_count", result.get("model", "unknown"))
             
-            conn = sqlite3.connect(str(DB_PATH))
-            cursor = conn.cursor()
-            cursor.execute("""
-                INSERT INTO sentiment_analysis 
-                (text, sentiment_label, confidence, model_used, analysis_type, symbol, scores)
-                VALUES (?, ?, ?, ?, ?, ?, ?)
-            """, (
-                text[:500],
-                sentiment_label,
-                confidence,
-                f"{model_used} models" if isinstance(model_used, int) else str(model_used),
-                mode,
-                symbol,
-                json.dumps(result.get("scores", {}))
-            ))
-            conn.commit()
-            conn.close()
+            # Save to database
+            try:
+                conn = sqlite3.connect(str(DB_PATH))
+                cursor = conn.cursor()
+                cursor.execute("""
+                    INSERT INTO sentiment_analysis 
+                    (text, sentiment_label, confidence, model_used, analysis_type, symbol, scores)
+                    VALUES (?, ?, ?, ?, ?, ?, ?)
+                """, (
+                    text[:500],
+                    sentiment_label,
+                    confidence,
+                    f"{model_used} models" if isinstance(model_used, int) else str(model_used),
+                    mode,
+                    symbol,
+                    json.dumps(result.get("scores", {}))
+                ))
+                conn.commit()
+                conn.close()
+                saved_to_db = True
+            except Exception as db_error:
+                logger.warning(f"Failed to save to database: {db_error}")
+                saved_to_db = False
             
             return {
                 "success": True,
+                "available": True,
                 "sentiment": sentiment_label,
                 "confidence": confidence,
                 "mode": mode,
                 "result": result,
-                "saved_to_db": True
+                "saved_to_db": saved_to_db
             }
             
         except ModelNotAvailable as e:
-            raise HTTPException(status_code=503, detail=f"Models not available: {str(e)}")
+            return {
+                "success": False,
+                "available": False,
+                "sentiment": "neutral",
+                "confidence": 0.0,
+                "mode": mode,
+                "error": str(e),
+                "reason": "model_unavailable"
+            }
             
     except HTTPException:
         raise
@@ -1110,40 +1140,74 @@ async def analyze_news(request: Dict[str, Any]):
             
             sentiment_label = result.get("sentiment", "neutral")
             sentiment_confidence = result.get("sentiment_confidence", 0.5)
+            sentiment_details = result.get("sentiment_details", {})
             related_symbols = request.get("related_symbols", [])
             
-            conn = sqlite3.connect(str(DB_PATH))
-            cursor = conn.cursor()
-            cursor.execute("""
-                INSERT INTO news_articles 
-                (title, content, url, source, sentiment_label, sentiment_confidence, related_symbols, published_date)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-            """, (
-                title[:500],
-                content[:2000] if content else None,
-                url,
-                source,
-                sentiment_label,
-                sentiment_confidence,
-                json.dumps(related_symbols) if related_symbols else None,
-                published_date
-            ))
-            conn.commit()
-            conn.close()
+            # Check if models were available
+            available = sentiment_details.get("available", True) if isinstance(sentiment_details, dict) else True
+            
+            if not available:
+                return {
+                    "success": False,
+                    "available": False,
+                    "news": {
+                        "title": title,
+                        "sentiment": "neutral",
+                        "confidence": 0.0,
+                        "error": sentiment_details.get("error", "Models not available")
+                    },
+                    "reason": "model_unavailable"
+                }
+            
+            # Save to database
+            try:
+                conn = sqlite3.connect(str(DB_PATH))
+                cursor = conn.cursor()
+                cursor.execute("""
+                    INSERT INTO news_articles 
+                    (title, content, url, source, sentiment_label, sentiment_confidence, related_symbols, published_date)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """, (
+                    title[:500],
+                    content[:2000] if content else None,
+                    url,
+                    source,
+                    sentiment_label,
+                    sentiment_confidence,
+                    json.dumps(related_symbols) if related_symbols else None,
+                    published_date
+                ))
+                conn.commit()
+                conn.close()
+                saved_to_db = True
+            except Exception as db_error:
+                logger.warning(f"Failed to save to database: {db_error}")
+                saved_to_db = False
             
             return {
                 "success": True,
+                "available": True,
                 "news": {
                     "title": title,
                     "sentiment": sentiment_label,
                     "confidence": sentiment_confidence,
-                    "details": result.get("sentiment_details", {})
+                    "details": sentiment_details
                 },
-                "saved_to_db": True
+                "saved_to_db": saved_to_db
             }
             
         except ModelNotAvailable as e:
-            raise HTTPException(status_code=503, detail=f"Models not available: {str(e)}")
+            return {
+                "success": False,
+                "available": False,
+                "news": {
+                    "title": title,
+                    "sentiment": "neutral",
+                    "confidence": 0.0,
+                    "error": str(e)
+                },
+                "reason": "model_unavailable"
+            }
             
     except HTTPException:
         raise
@@ -1387,6 +1451,7 @@ async def predict_with_model(model_key: str, request: Dict[str, Any]):
             
             return {
                 "success": True,
+                "available": True,
                 "model_key": model_key,
                 "model_id": spec.model_id,
                 "task": spec.task,
@@ -1395,7 +1460,14 @@ async def predict_with_model(model_key: str, request: Dict[str, Any]):
                 "timestamp": datetime.now().isoformat()
             }
         except ModelNotAvailable as e:
-            raise HTTPException(status_code=503, detail=f"Model not available: {str(e)}")
+            return {
+                "success": False,
+                "available": False,
+                "model_key": model_key,
+                "model_id": spec.model_id,
+                "error": str(e),
+                "reason": "model_unavailable"
+            }
             
     except HTTPException:
         raise
@@ -1601,12 +1673,18 @@ async def run_hf_sentiment(data: Dict[str, Any]):
             all_results = []
             total_vote = 0.0
             count = 0
+            models_available = False
             
             for text in texts:
                 if not text.strip():
                     continue
                 
                 result = analyze_market_text(text.strip())
+                
+                # Check if models are available
+                if result.get("available", True):
+                    models_available = True
+                
                 label = result.get("label", "neutral")
                 confidence = result.get("confidence", 0.5)
                 
@@ -1623,12 +1701,14 @@ async def run_hf_sentiment(data: Dict[str, Any]):
                     "text": text[:100],
                     "label": label,
                     "confidence": confidence,
-                    "vote": vote_score
+                    "vote": vote_score,
+                    "available": result.get("available", True)
                 })
             
             avg_vote = total_vote / count if count > 0 else 0.0
             
             return {
+                "available": models_available,
                 "vote": avg_vote,
                 "results": all_results,
                 "count": count,
@@ -1636,7 +1716,15 @@ async def run_hf_sentiment(data: Dict[str, Any]):
             }
             
         except ModelNotAvailable as e:
-            raise HTTPException(status_code=503, detail=f"Models not available: {str(e)}")
+            return {
+                "available": False,
+                "vote": 0.0,
+                "results": [],
+                "count": 0,
+                "average_confidence": 0.0,
+                "error": str(e),
+                "reason": "model_unavailable"
+            }
             
     except HTTPException:
         raise

index.html

@@ -139,6 +139,30 @@
                     <h2>تحلیل احساسات</h2>
                 </div>
                 
+                <!-- Global Market Sentiment -->
+                <div class="card">
+                    <h3>احساسات کلی بازار</h3>
+                    <p style="color: var(--text-secondary); margin-bottom: 15px;">تحلیل احساسات کلی بازار رمز ارز با استفاده از مدل‌های AI</p>
+                    <button class="btn-primary" onclick="analyzeGlobalSentiment()">📊 تحلیل احساسات بازار</button>
+                    <div id="global-sentiment-result" style="margin-top: 20px;"></div>
+                </div>
+                
+                <!-- Per-Asset Sentiment -->
+                <div class="card">
+                    <h3>تحلیل احساسات برای هر ارز</h3>
+                    <div class="form-group">
+                        <label>نماد ارز (مثال: BTC, ETH):</label>
+                        <input type="text" id="asset-symbol" placeholder="BTC" style="text-transform: uppercase;">
+                    </div>
+                    <div class="form-group">
+                        <label>متن یا خبر مرتبط (اختیاری):</label>
+                        <textarea id="asset-sentiment-text" rows="3" placeholder="مثال: Bitcoin breaks resistance at $50,000"></textarea>
+                    </div>
+                    <button class="btn-primary" onclick="analyzeAssetSentiment()">🔍 تحلیل احساسات ارز</button>
+                    <div id="asset-sentiment-result" style="margin-top: 20px;"></div>
+                </div>
+                
+                <!-- Text Analysis -->
                 <div class="card">
                     <h3>تحلیل متن</h3>
                     <div class="form-group">
@@ -146,13 +170,37 @@
                         <textarea id="sentiment-text" rows="5" placeholder="مثال: Bitcoin price is rising rapidly!"></textarea>
                     </div>
                     <div class="form-group">
-                        <label>انتخاب مدل:</label>
+                        <label>نوع تحلیل:</label>
+                        <select id="sentiment-mode">
+                            <option value="auto">خودکار (Crypto)</option>
+                            <option value="crypto">رمز ارز (Crypto)</option>
+                            <option value="financial">مالی (Financial)</option>
+                            <option value="social">اجتماعی (Social)</option>
+                        </select>
+                    </div>
+                    <div class="form-group">
+                        <label>انتخاب مدل (اختیاری):</label>
                         <select id="sentiment-model"></select>
                     </div>
                     <button class="btn-primary" onclick="analyzeSentiment()">🔍 تحلیل</button>
                     <div id="sentiment-result"></div>
                 </div>
                 
+                <!-- News/Financial Sentiment -->
+                <div class="card">
+                    <h3>تحلیل احساسات اخبار و مالی</h3>
+                    <div class="form-group">
+                        <label>عنوان خبر:</label>
+                        <input type="text" id="news-title" placeholder="مثال: Bitcoin ETF Approval Expected">
+                    </div>
+                    <div class="form-group">
+                        <label>محتوا یا توضیحات:</label>
+                        <textarea id="news-content" rows="4" placeholder="متن کامل خبر یا توضیحات..."></textarea>
+                    </div>
+                    <button class="btn-primary" onclick="analyzeNewsSentiment()">📰 تحلیل خبر</button>
+                    <div id="news-sentiment-result" style="margin-top: 20px;"></div>
+                </div>
+                
                 <div class="card">
                     <h3>تاریخچه تحلیل‌ها</h3>
                     <div id="sentiment-history"></div>

static/css/main.css

@@ -406,6 +406,154 @@ table tr:hover {
     text-decoration: underline;
 }
 
+/* Sentiment Badges */
+.sentiment-badge {
+    display: inline-block;
+    padding: 6px 12px;
+    border-radius: 8px;
+    font-size: 13px;
+    font-weight: 600;
+    margin: 5px 5px 5px 0;
+}
+
+.sentiment-badge.bullish {
+    background: rgba(16, 185, 129, 0.2);
+    color: var(--success);
+    border: 1px solid rgba(16, 185, 129, 0.3);
+}
+
+.sentiment-badge.bearish {
+    background: rgba(239, 68, 68, 0.2);
+    color: var(--danger);
+    border: 1px solid rgba(239, 68, 68, 0.3);
+}
+
+.sentiment-badge.neutral {
+    background: rgba(156, 163, 175, 0.2);
+    color: var(--text-secondary);
+    border: 1px solid rgba(156, 163, 175, 0.3);
+}
+
+/* AI Result Cards */
+.ai-result-card {
+    background: rgba(17, 24, 39, 0.6);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 20px;
+    margin-top: 15px;
+    transition: all 0.3s;
+}
+
+.ai-result-card:hover {
+    border-color: var(--primary);
+    box-shadow: 0 5px 15px rgba(102, 126, 234, 0.2);
+}
+
+.ai-result-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 15px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid var(--border);
+}
+
+.ai-result-metric {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    padding: 15px;
+    background: rgba(31, 41, 55, 0.6);
+    border-radius: 10px;
+    min-width: 120px;
+}
+
+.ai-result-metric-value {
+    font-size: 28px;
+    font-weight: 800;
+    margin-bottom: 5px;
+}
+
+.ai-result-metric-label {
+    font-size: 12px;
+    color: var(--text-secondary);
+    text-transform: uppercase;
+}
+
+/* Model Status Indicators */
+.model-status {
+    display: inline-flex;
+    align-items: center;
+    gap: 6px;
+    padding: 4px 10px;
+    border-radius: 6px;
+    font-size: 12px;
+    font-weight: 600;
+}
+
+.model-status.available {
+    background: rgba(16, 185, 129, 0.15);
+    color: var(--success);
+}
+
+.model-status.unavailable {
+    background: rgba(239, 68, 68, 0.15);
+    color: var(--danger);
+}
+
+.model-status.partial {
+    background: rgba(245, 158, 11, 0.15);
+    color: var(--warning);
+}
+
+/* Form Improvements for AI Sections */
+.form-group input[type="text"] {
+    text-transform: uppercase;
+}
+
+.form-group textarea {
+    resize: vertical;
+    min-height: 80px;
+}
+
+/* Loading States */
+.loading {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    padding: 40px;
+    color: var(--text-secondary);
+}
+
+.loading .spinner {
+    margin-bottom: 15px;
+}
+
+/* Confidence Bar */
+.confidence-bar {
+    width: 100%;
+    height: 8px;
+    background: rgba(31, 41, 55, 0.6);
+    border-radius: 4px;
+    overflow: hidden;
+    margin-top: 5px;
+}
+
+.confidence-fill {
+    height: 100%;
+    background: linear-gradient(90deg, var(--primary), var(--primary-dark));
+    transition: width 0.3s ease;
+}
+
+.confidence-fill.high {
+    background: linear-gradient(90deg, var(--success), #059669);
+}
+
+.confidence-fill.low {
+    background: linear-gradient(90deg, var(--danger), #dc2626);
+}
+
 /* Responsive */
 @media (max-width: 768px) {
     .header-content {
@@ -428,5 +576,18 @@ table tr:hover {
     .stats-grid {
         grid-template-columns: 1fr;
     }
+    
+    .ai-result-metric {
+        min-width: 100px;
+        padding: 10px;
+    }
+    
+    .ai-result-metric-value {
+        font-size: 20px;
+    }
+    
+    .card {
+        padding: 15px;
+    }
 }
 

static/js/app.js

@@ -514,9 +514,213 @@ async function loadSentimentModels() {
     }
 }
 
-// Analyze Sentiment
+// Analyze Global Market Sentiment
+async function analyzeGlobalSentiment() {
+    const resultDiv = document.getElementById('global-sentiment-result');
+    resultDiv.innerHTML = '<div class="loading"><div class="spinner"></div> در حال تحلیل احساسات بازار...</div>';
+    
+    try {
+        // Use market text analysis with sample market-related text
+        const marketText = "Cryptocurrency market analysis: Bitcoin, Ethereum, and major altcoins showing mixed signals. Market sentiment analysis required.";
+        
+        const response = await fetch('/api/sentiment/analyze', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ text: marketText, mode: 'crypto' })
+        });
+        
+        const data = await response.json();
+        
+        if (!data.available) {
+            resultDiv.innerHTML = `
+                <div class="alert alert-warning">
+                    <strong>⚠️ مدل‌ها در دسترس نیستند:</strong> ${data.error || 'مدل‌های AI در حال حاضر در دسترس نیستند'}
+                </div>
+            `;
+            return;
+        }
+        
+        const sentiment = data.sentiment || 'neutral';
+        const confidence = data.confidence || 0;
+        const sentimentEmoji = sentiment === 'bullish' ? '📈' : sentiment === 'bearish' ? '📉' : '➡️';
+        const sentimentColor = sentiment === 'bullish' ? 'var(--success)' : sentiment === 'bearish' ? 'var(--danger)' : 'var(--text-secondary)';
+        
+        resultDiv.innerHTML = `
+            <div class="alert alert-success" style="border-left: 4px solid ${sentimentColor};">
+                <h4 style="margin-bottom: 15px;">احساسات کلی بازار</h4>
+                <div style="display: grid; gap: 10px;">
+                    <div style="text-align: center; padding: 20px;">
+                        <div style="font-size: 48px; margin-bottom: 10px;">${sentimentEmoji}</div>
+                        <div style="font-size: 24px; font-weight: 700; color: ${sentimentColor}; margin-bottom: 5px;">
+                            ${sentiment === 'bullish' ? 'صعودی' : sentiment === 'bearish' ? 'نزولی' : 'خنثی'}
+                        </div>
+                        <div style="color: var(--text-secondary);">
+                            اعتماد: ${(confidence * 100).toFixed(1)}%
+                        </div>
+                    </div>
+                    <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid var(--border);">
+                        <strong>جزئیات:</strong>
+                        <div style="margin-top: 5px; font-size: 13px; color: var(--text-secondary);">
+                            این تحلیل بر اساس مدل‌های AI انجام شده است.
+                        </div>
+                    </div>
+                </div>
+            </div>
+        `;
+    } catch (error) {
+        console.error('Global sentiment analysis error:', error);
+        resultDiv.innerHTML = `<div class="alert alert-error">خطا در تحلیل: ${error.message}</div>`;
+        showError('خطا در تحلیل احساسات بازار');
+    }
+}
+
+// Analyze Asset Sentiment
+async function analyzeAssetSentiment() {
+    const symbol = document.getElementById('asset-symbol').value.trim().toUpperCase();
+    const text = document.getElementById('asset-sentiment-text').value.trim();
+    
+    if (!symbol) {
+        showError('لطفاً نماد ارز را وارد کنید');
+        return;
+    }
+    
+    const resultDiv = document.getElementById('asset-sentiment-result');
+    resultDiv.innerHTML = '<div class="loading"><div class="spinner"></div> در حال تحلیل...</div>';
+    
+    try {
+        // Use provided text or default text with symbol
+        const analysisText = text || `${symbol} market analysis and sentiment`;
+        
+        const response = await fetch('/api/sentiment/analyze', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ text: analysisText, mode: 'crypto', symbol: symbol })
+        });
+        
+        const data = await response.json();
+        
+        if (!data.available) {
+            resultDiv.innerHTML = `
+                <div class="alert alert-warning">
+                    <strong>⚠️ مدل‌ها در دسترس نیستند:</strong> ${data.error || 'مدل‌های AI در حال حاضر در دسترس نیستند'}
+                </div>
+            `;
+            return;
+        }
+        
+        const sentiment = data.sentiment || 'neutral';
+        const confidence = data.confidence || 0;
+        const sentimentEmoji = sentiment === 'bullish' ? '📈' : sentiment === 'bearish' ? '📉' : '➡️';
+        const sentimentColor = sentiment === 'bullish' ? 'var(--success)' : sentiment === 'bearish' ? 'var(--danger)' : 'var(--text-secondary)';
+        
+        resultDiv.innerHTML = `
+            <div class="alert alert-success" style="border-left: 4px solid ${sentimentColor};">
+                <h4 style="margin-bottom: 15px;">نتیجه تحلیل احساسات ${symbol}</h4>
+                <div style="display: grid; gap: 10px;">
+                    <div>
+                        <strong>احساسات:</strong> 
+                        <span style="color: ${sentimentColor}; font-weight: 700; font-size: 18px;">
+                            ${sentimentEmoji} ${sentiment === 'bullish' ? 'صعودی' : sentiment === 'bearish' ? 'نزولی' : 'خنثی'}
+                        </span>
+                    </div>
+                    <div>
+                        <strong>اعتماد:</strong> 
+                        <span style="color: var(--primary); font-weight: 600;">
+                            ${(confidence * 100).toFixed(2)}%
+                        </span>
+                    </div>
+                    ${text ? `
+                        <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid var(--border);">
+                            <strong>متن تحلیل شده:</strong>
+                            <div style="margin-top: 5px; padding: 10px; background: rgba(31, 41, 55, 0.6); border-radius: 5px; font-size: 13px; color: var(--text-secondary);">
+                                "${text.substring(0, 200)}${text.length > 200 ? '...' : ''}"
+                            </div>
+                        </div>
+                    ` : ''}
+                </div>
+            </div>
+        `;
+    } catch (error) {
+        console.error('Asset sentiment analysis error:', error);
+        resultDiv.innerHTML = `<div class="alert alert-error">خطا در تحلیل: ${error.message}</div>`;
+        showError('خطا در تحلیل احساسات ارز');
+    }
+}
+
+// Analyze News Sentiment
+async function analyzeNewsSentiment() {
+    const title = document.getElementById('news-title').value.trim();
+    const content = document.getElementById('news-content').value.trim();
+    
+    if (!title && !content) {
+        showError('لطفاً عنوان یا محتوای خبر را وارد کنید');
+        return;
+    }
+    
+    const resultDiv = document.getElementById('news-sentiment-result');
+    resultDiv.innerHTML = '<div class="loading"><div class="spinner"></div> در حال تحلیل...</div>';
+    
+    try {
+        const response = await fetch('/api/news/analyze', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ title: title, content: content, description: content })
+        });
+        
+        const data = await response.json();
+        
+        if (!data.available) {
+            resultDiv.innerHTML = `
+                <div class="alert alert-warning">
+                    <strong>⚠️ مدل‌ها در دسترس نیستند:</strong> ${data.news?.error || data.error || 'مدل‌های AI در حال حاضر در دسترس نیستند'}
+                </div>
+            `;
+            return;
+        }
+        
+        const newsData = data.news || {};
+        const sentiment = newsData.sentiment || 'neutral';
+        const confidence = newsData.confidence || 0;
+        const sentimentEmoji = sentiment === 'bullish' || sentiment === 'positive' ? '📈' : 
+                              sentiment === 'bearish' || sentiment === 'negative' ? '📉' : '➡️';
+        const sentimentColor = sentiment === 'bullish' || sentiment === 'positive' ? 'var(--success)' : 
+                              sentiment === 'bearish' || sentiment === 'negative' ? 'var(--danger)' : 'var(--text-secondary)';
+        
+        resultDiv.innerHTML = `
+            <div class="alert alert-success" style="border-left: 4px solid ${sentimentColor};">
+                <h4 style="margin-bottom: 15px;">نتیجه تحلیل خبر</h4>
+                <div style="display: grid; gap: 10px;">
+                    <div>
+                        <strong>عنوان:</strong> 
+                        <span style="color: var(--text-primary);">${title || 'بدون عنوان'}</span>
+                    </div>
+                    <div>
+                        <strong>احساسات:</strong> 
+                        <span style="color: ${sentimentColor}; font-weight: 700; font-size: 18px;">
+                            ${sentimentEmoji} ${sentiment === 'bullish' || sentiment === 'positive' ? 'مثبت' : 
+                              sentiment === 'bearish' || sentiment === 'negative' ? 'منفی' : 'خنثی'}
+                        </span>
+                    </div>
+                    <div>
+                        <strong>اعتماد:</strong> 
+                        <span style="color: var(--primary); font-weight: 600;">
+                            ${(confidence * 100).toFixed(2)}%
+                        </span>
+                    </div>
+                </div>
+            </div>
+        `;
+    } catch (error) {
+        console.error('News sentiment analysis error:', error);
+        resultDiv.innerHTML = `<div class="alert alert-error">خطا در تحلیل: ${error.message}</div>`;
+        showError('خطا در تحلیل خبر');
+    }
+}
+
+// Analyze Sentiment (updated)
 async function analyzeSentiment() {
     const text = document.getElementById('sentiment-text').value;
+    const mode = document.getElementById('sentiment-mode').value;
     const modelKey = document.getElementById('sentiment-model').value;
     
     if (!text.trim()) {
@@ -528,60 +732,35 @@ async function analyzeSentiment() {
     resultDiv.innerHTML = '<div class="loading"><div class="spinner"></div> در حال تحلیل...</div>';
     
     try {
-        // Try multiple endpoints
         let response;
-        let endpoint = '/api/hf/run-sentiment';
         
-        // If model key is selected, try model-specific endpoint
-        if (modelKey) {
-            try {
-                endpoint = `/api/models/${modelKey}/predict`;
-                response = await fetch(endpoint, {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({ text: text })
-                });
-            } catch {
-                // Fallback to general sentiment endpoint
-                endpoint = '/api/hf/run-sentiment';
-                response = await fetch(endpoint, {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({ texts: [text], model_key: modelKey })
-                });
-            }
-        } else {
-            response = await fetch('/api/hf/run-sentiment', {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({ texts: [text] })
-            });
-        }
+        // Use the sentiment/analyze endpoint with mode
+        response = await fetch('/api/sentiment/analyze', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ text: text, mode: mode })
+        });
         
         const data = await response.json();
         
-        // Handle different response formats
-        let result;
-        if (data.results && data.results.length > 0) {
-            result = data.results[0];
-        } else if (data.result) {
-            result = data.result;
-        } else if (data.label || data.sentiment) {
-            result = data;
-        } else {
-            throw new Error('فرمت پاسخ نامعتبر است');
+        if (!data.available) {
+            resultDiv.innerHTML = `
+                <div class="alert alert-warning">
+                    <strong>⚠️ مدل‌ها در دسترس نیستند:</strong> ${data.error || 'مدل‌های AI در حال حاضر در دسترس نیستند'}
+                </div>
+            `;
+            return;
         }
         
-        const label = result.label || result.sentiment || 'unknown';
-        const confidence = result.confidence || result.score || 0;
-        const vote = result.vote || 0;
-        const modelUsed = result.model || modelKey || 'default';
+        const label = data.sentiment || 'neutral';
+        const confidence = data.confidence || 0;
+        const result = data.result || {};
         
         // Determine sentiment emoji and color
-        const sentimentEmoji = label.toUpperCase().includes('POSITIVE') || label.toUpperCase().includes('BULLISH') ? '📈' :
-                              label.toUpperCase().includes('NEGATIVE') || label.toUpperCase().includes('BEARISH') ? '📉' : '➡️';
-        const sentimentColor = label.toUpperCase().includes('POSITIVE') || label.toUpperCase().includes('BULLISH') ? 'var(--success)' :
-                              label.toUpperCase().includes('NEGATIVE') || label.toUpperCase().includes('BEARISH') ? 'var(--danger)' : 'var(--text-secondary)';
+        const sentimentEmoji = label === 'bullish' || label === 'positive' ? '📈' :
+                              label === 'bearish' || label === 'negative' ? '📉' : '➡️';
+        const sentimentColor = label === 'bullish' || label === 'positive' ? 'var(--success)' :
+                              label === 'bearish' || label === 'negative' ? 'var(--danger)' : 'var(--text-secondary)';
         
         resultDiv.innerHTML = `
             <div class="alert alert-success" style="margin-top: 20px; border-left: 4px solid ${sentimentColor};">
@@ -590,7 +769,8 @@ async function analyzeSentiment() {
                     <div>
                         <strong>احساسات:</strong> 
                         <span style="color: ${sentimentColor}; font-weight: 700; font-size: 18px;">
-                            ${sentimentEmoji} ${label}
+                            ${sentimentEmoji} ${label === 'bullish' || label === 'positive' ? 'صعودی/مثبت' : 
+                              label === 'bearish' || label === 'negative' ? 'نزولی/منفی' : 'خنثی'}
                         </span>
                     </div>
                     <div>
@@ -599,17 +779,9 @@ async function analyzeSentiment() {
                             ${(confidence * 100).toFixed(2)}%
                         </span>
                     </div>
-                    ${vote !== undefined ? `
-                        <div>
-                            <strong>رأی مدل:</strong> 
-                            <span style="color: ${vote > 0 ? 'var(--success)' : vote < 0 ? 'var(--danger)' : 'var(--text-secondary)'}; font-weight: 600;">
-                                ${vote > 0 ? '↑' : vote < 0 ? '↓' : '→'} ${vote.toFixed(2)}
-                            </span>
-                        </div>
-                    ` : ''}
                     <div>
-                        <strong>مدل استفاده شده:</strong> 
-                        <span style="font-family: monospace; font-size: 12px; color: var(--text-secondary);">${modelUsed}</span>
+                        <strong>نوع تحلیل:</strong> 
+                        <span style="color: var(--text-secondary);">${mode}</span>
                     </div>
                     <div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid var(--border);">
                         <strong>متن تحلیل شده:</strong>
@@ -626,7 +798,7 @@ async function analyzeSentiment() {
             text: text.substring(0, 100),
             label: label,
             confidence: confidence,
-            model: modelUsed,
+            model: mode,
             timestamp: new Date().toISOString()
         });