Upload folder using huggingface_hub

airflow/Dockerfile

@@ -0,0 +1,16 @@
+# Astro CLI Dockerfile for ModelX-Ultimate Airflow
+# Includes all dependencies for ML pipeline execution
+
+FROM quay.io/astronomer/astro-runtime:12.3.0
+
+# Install system dependencies
+USER root
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+USER astro
+
+# Copy requirements and install
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt

airflow/README.md

@@ -0,0 +1,69 @@
+# ModelX-Ultimate Astro Airflow
+
+Centralized Apache Airflow setup using Astronomer's Astro CLI for managing all ML pipelines.
+
+## DAGs Overview
+
+| DAG | Schedule | Description |
+|-----|----------|-------------|
+| `weather_prediction_daily` | 4:00 AM IST | LSTM model for 25 Sri Lankan districts |
+| `currency_prediction_daily` | 4:05 AM IST | GRU model for USD/LKR forex |
+| `stock_prediction_daily` | 4:15 AM IST | BiLSTM models for 10 stocks |
+| `anomaly_detection_periodic` | Every 6h | Anomaly detection retraining |
+
+## Quick Start
+
+### 1. Install Astro CLI
+```bash
+# macOS
+brew install astro
+
+# Windows (PowerShell as Admin)
+winget install -e --id Astronomer.Astro
+
+# Linux
+curl -sSL install.astronomer.io | sudo bash -s
+```
+
+### 2. Start Airflow
+```bash
+cd airflow
+astro dev start
+```
+
+### 3. Access Airflow UI
+- URL: http://localhost:8080
+- Username: `admin`
+- Password: `admin`
+
+### 4. Enable DAGs
+Turn on the DAGs in the Airflow UI to start scheduled runs.
+
+## Directory Structure
+```
+airflow/
+├── dags/
+│   ├── weather_prediction_dag.py
+│   ├── currency_prediction_dag.py
+│   ├── stock_prediction_dag.py
+│   └── anomaly_detection_dag.py
+├── Dockerfile
+├── requirements.txt
+├── airflow.env.example
+└── README.md
+```
+
+## Manual Trigger
+```bash
+# Trigger a specific DAG
+astro dev run dags trigger weather_prediction_daily
+```
+
+## Logs
+```bash
+# View scheduler logs
+astro dev logs --scheduler
+
+# View webserver logs
+astro dev logs --webserver
+```

airflow/airflow.env.example

@@ -0,0 +1,36 @@
+# ModelX-Ultimate Airflow Configuration
+# =====================================
+# Centralized Astro Apache Airflow setup for all ML pipelines
+#
+# Pipelines managed:
+# - Weather Prediction (4:00 AM IST daily)
+# - Currency Prediction (4:00 AM IST daily)
+# - Stock Prediction (4:00 AM IST daily)
+# - Anomaly Detection (continuous)
+
+# Airflow settings
+AIRFLOW_UID=50000
+AIRFLOW_GID=0
+
+# Environment
+AIRFLOW__CORE__EXECUTOR=LocalExecutor
+AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
+AIRFLOW__CORE__FERNET_KEY=
+AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
+AIRFLOW__CORE__LOAD_EXAMPLES=false
+AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
+
+# Webserver
+AIRFLOW__WEBSERVER__SECRET_KEY=modelx-secret-key-change-in-production
+AIRFLOW__WEBSERVER__EXPOSE_CONFIG=false
+
+# Scheduler
+AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK=true
+AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=30
+
+# Timezone
+AIRFLOW__CORE__DEFAULT_TIMEZONE=Asia/Colombo
+
+# Project paths
+PROJECT_ROOT=/opt/airflow
+MODELS_PATH=/opt/airflow/models

airflow/dags/anomaly_detection_dag.py

@@ -0,0 +1,73 @@
+"""
+Anomaly Detection DAG
+Runs every 6 hours
+Retrains anomaly detection model on latest data
+"""
+import os
+import sys
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+
+# Project paths
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+ANOMALY_MODEL_PATH = PROJECT_ROOT / "models" / "anomaly-detection"
+
+default_args = {
+    "owner": "modelx",
+    "depends_on_past": False,
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 1,
+    "retry_delay": timedelta(minutes=5),
+}
+
+
+def run_anomaly_training(**context):
+    """Run the anomaly detection training pipeline."""
+    main_py = ANOMALY_MODEL_PATH / "main.py"
+    
+    if not main_py.exists():
+        raise FileNotFoundError(f"Anomaly training script not found: {main_py}")
+    
+    result = subprocess.run(
+        [sys.executable, str(main_py)],
+        capture_output=True,
+        text=True,
+        cwd=str(ANOMALY_MODEL_PATH)
+    )
+    
+    print("STDOUT:", result.stdout[-2000:] if len(result.stdout) > 2000 else result.stdout)
+    if result.stderr:
+        print("STDERR:", result.stderr[-1000:])
+    
+    if result.returncode != 0:
+        raise Exception(f"Anomaly training failed with exit code {result.returncode}")
+    
+    return True
+
+
+with DAG(
+    dag_id="anomaly_detection_periodic",
+    default_args=default_args,
+    description="Periodic anomaly detection model retraining",
+    schedule_interval="0 */6 * * *",  # Every 6 hours
+    start_date=datetime(2024, 12, 1),
+    catchup=False,
+    tags=["anomaly", "ml", "detection", "periodic"],
+    max_active_runs=1,
+) as dag:
+    
+    train_anomaly = PythonOperator(
+        task_id="train_anomaly_model",
+        python_callable=run_anomaly_training,
+        provide_context=True,
+        execution_timeout=timedelta(minutes=30),
+    )
+
+
+if __name__ == "__main__":
+    print(f"Anomaly Detection DAG - Schedule: Every 6 hours")

airflow/dags/currency_prediction_dag.py

@@ -0,0 +1,73 @@
+"""
+Currency Prediction DAG
+Runs daily at 4:00 AM IST (22:30 UTC)
+Trains GRU model for USD/LKR exchange rate prediction
+"""
+import os
+import sys
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+
+# Project paths
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+CURRENCY_MODEL_PATH = PROJECT_ROOT / "models" / "currency-volatility-prediction"
+
+default_args = {
+    "owner": "modelx",
+    "depends_on_past": False,
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 2,
+    "retry_delay": timedelta(minutes=10),
+}
+
+
+def run_currency_training(**context):
+    """Run the currency prediction training pipeline."""
+    main_py = CURRENCY_MODEL_PATH / "main.py"
+    
+    if not main_py.exists():
+        raise FileNotFoundError(f"Currency training script not found: {main_py}")
+    
+    result = subprocess.run(
+        [sys.executable, str(main_py), "--mode", "full"],
+        capture_output=True,
+        text=True,
+        cwd=str(CURRENCY_MODEL_PATH)
+    )
+    
+    print("STDOUT:", result.stdout[-3000:] if len(result.stdout) > 3000 else result.stdout)
+    if result.stderr:
+        print("STDERR:", result.stderr[-1000:])
+    
+    if result.returncode != 0:
+        raise Exception(f"Currency training failed with exit code {result.returncode}")
+    
+    return True
+
+
+with DAG(
+    dag_id="currency_prediction_daily",
+    default_args=default_args,
+    description="Daily USD/LKR exchange rate prediction using GRU model",
+    schedule_interval="35 22 * * *",  # 4:05 AM IST = 22:35 UTC (staggered)
+    start_date=datetime(2024, 12, 1),
+    catchup=False,
+    tags=["currency", "ml", "prediction", "gru", "daily", "forex"],
+    max_active_runs=1,
+) as dag:
+    
+    train_currency = PythonOperator(
+        task_id="train_currency_model",
+        python_callable=run_currency_training,
+        provide_context=True,
+        execution_timeout=timedelta(hours=1),
+    )
+
+
+if __name__ == "__main__":
+    print(f"Currency Prediction DAG - Schedule: 4:05 AM IST daily")

airflow/dags/stock_prediction_dag.py

@@ -0,0 +1,73 @@
+"""
+Stock Prediction DAG
+Runs daily at 4:15 AM IST (22:45 UTC)
+Trains BiLSTM models for 10 stocks
+"""
+import os
+import sys
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+
+# Project paths
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+STOCK_MODEL_PATH = PROJECT_ROOT / "models" / "stock-price-prediction"
+
+default_args = {
+    "owner": "modelx",
+    "depends_on_past": False,
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 2,
+    "retry_delay": timedelta(minutes=10),
+}
+
+
+def run_stock_training(**context):
+    """Run the multi-stock training pipeline."""
+    main_py = STOCK_MODEL_PATH / "main.py"
+    
+    if not main_py.exists():
+        raise FileNotFoundError(f"Stock training script not found: {main_py}")
+    
+    result = subprocess.run(
+        [sys.executable, str(main_py)],
+        capture_output=True,
+        text=True,
+        cwd=str(STOCK_MODEL_PATH)
+    )
+    
+    print("STDOUT:", result.stdout[-5000:] if len(result.stdout) > 5000 else result.stdout)
+    if result.stderr:
+        print("STDERR:", result.stderr[-2000:])
+    
+    if result.returncode != 0:
+        raise Exception(f"Stock training failed with exit code {result.returncode}")
+    
+    return True
+
+
+with DAG(
+    dag_id="stock_prediction_daily",
+    default_args=default_args,
+    description="Daily stock prediction for 10 stocks using BiLSTM",
+    schedule_interval="45 22 * * *",  # 4:15 AM IST = 22:45 UTC (staggered)
+    start_date=datetime(2024, 12, 1),
+    catchup=False,
+    tags=["stock", "ml", "prediction", "lstm", "daily"],
+    max_active_runs=1,
+) as dag:
+    
+    train_stocks = PythonOperator(
+        task_id="train_all_stocks",
+        python_callable=run_stock_training,
+        provide_context=True,
+        execution_timeout=timedelta(hours=4),  # 10 stocks take time
+    )
+
+
+if __name__ == "__main__":
+    print(f"Stock Prediction DAG - Schedule: 4:15 AM IST daily")

airflow/dags/weather_prediction_dag.py

@@ -0,0 +1,74 @@
+"""
+Weather Prediction DAG
+Runs daily at 4:00 AM IST (22:30 UTC)
+Trains LSTM model for 25 Sri Lankan districts
+"""
+import os
+import sys
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+from airflow.operators.bash import BashOperator
+
+# Project paths
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+WEATHER_MODEL_PATH = PROJECT_ROOT / "models" / "weather-prediction"
+
+default_args = {
+    "owner": "modelx",
+    "depends_on_past": False,
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 2,
+    "retry_delay": timedelta(minutes=10),
+}
+
+
+def run_weather_training(**context):
+    """Run the weather prediction training pipeline."""
+    main_py = WEATHER_MODEL_PATH / "main.py"
+    
+    if not main_py.exists():
+        raise FileNotFoundError(f"Weather training script not found: {main_py}")
+    
+    result = subprocess.run(
+        [sys.executable, str(main_py), "--mode", "full"],
+        capture_output=True,
+        text=True,
+        cwd=str(WEATHER_MODEL_PATH)
+    )
+    
+    print("STDOUT:", result.stdout[-3000:] if len(result.stdout) > 3000 else result.stdout)
+    if result.stderr:
+        print("STDERR:", result.stderr[-1000:])
+    
+    if result.returncode != 0:
+        raise Exception(f"Weather training failed with exit code {result.returncode}")
+    
+    return True
+
+
+with DAG(
+    dag_id="weather_prediction_daily",
+    default_args=default_args,
+    description="Daily weather prediction model training for 25 Sri Lankan districts",
+    schedule_interval="30 22 * * *",  # 4:00 AM IST = 22:30 UTC
+    start_date=datetime(2024, 12, 1),
+    catchup=False,
+    tags=["weather", "ml", "prediction", "lstm", "daily"],
+    max_active_runs=1,
+) as dag:
+    
+    train_weather = PythonOperator(
+        task_id="train_weather_model",
+        python_callable=run_weather_training,
+        provide_context=True,
+        execution_timeout=timedelta(hours=2),
+    )
+
+
+if __name__ == "__main__":
+    print(f"Weather Prediction DAG - Schedule: 4:00 AM IST daily")

airflow/requirements.txt

@@ -0,0 +1,23 @@
+# ModelX-Ultimate Airflow Requirements
+# ML Pipeline Dependencies
+
+# Core ML
+tensorflow>=2.15.0
+scikit-learn>=1.3.0
+numpy>=1.24.0
+pandas>=2.0.0
+
+# Stock data
+yfinance>=0.2.36
+
+# MLflow tracking
+mlflow>=2.9.0
+
+# Data processing
+joblib>=1.3.0
+
+# HTTP requests (for API calls)
+requests>=2.31.0
+
+# Environment management
+python-dotenv>=1.0.0

frontend/app/components/dashboard/DashboardOverview.tsx

@@ -190,29 +190,43 @@ const DashboardOverview = () => {
         </div>
       </Card>
 
-      {/* Operational Risk Radar */}
-      <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
-        <Card className="p-6 bg-card border-border">
-          <Cloud className="w-8 h-8 text-warning mb-3" />
-          <p className="text-2xl font-bold">{Math.round(dashboard.logistics_friction * 100)}%</p>
-          <p className="text-xs text-muted-foreground uppercase">Logistics Friction</p>
-        </Card>
-        <Card className="p-6 bg-card border-border">
-          <AlertTriangle className="w-8 h-8 text-destructive mb-3" />
-          <p className="text-2xl font-bold">{Math.round(dashboard.compliance_volatility * 100)}%</p>
-          <p className="text-xs text-muted-foreground uppercase">Compliance Volatility</p>
-        </Card>
-        <Card className="p-6 bg-card border-border">
-          <TrendingUp className="w-8 h-8 text-info mb-3" />
-          <p className="text-2xl font-bold">{Math.round(dashboard.market_instability * 100)}%</p>
-          <p className="text-xs text-muted-foreground uppercase">Market Instability</p>
-        </Card>
-        <Card className="p-6 bg-card border-border">
-          <Building className="w-8 h-8 text-success mb-3" />
-          <p className="text-2xl font-bold">{Math.round(dashboard.opportunity_index * 100)}%</p>
-          <p className="text-xs text-muted-foreground uppercase">Opportunity Index</p>
-        </Card>
-      </div>
+      {/* Operational Risk Indicators - Computed from Events */}
+      {sortedEvents.length > 0 && (
+        <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
+          <Card className="p-6 bg-card border-border">
+            <Cloud className="w-8 h-8 text-warning mb-3" />
+            <p className="text-2xl font-bold">
+              {Math.min(100, Math.round(
+                (sortedEvents.filter(e => e.domain === 'meteorological' || e.summary?.toLowerCase().includes('weather')).length / Math.max(sortedEvents.length, 1)) * 100 * 3
+              ))}%
+            </p>
+            <p className="text-xs text-muted-foreground uppercase">Weather Impact</p>
+          </Card>
+          <Card className="p-6 bg-card border-border">
+            <AlertTriangle className="w-8 h-8 text-destructive mb-3" />
+            <p className="text-2xl font-bold">
+              {Math.round((criticalEvents.length / Math.max(sortedEvents.length, 1)) * 100)}%
+            </p>
+            <p className="text-xs text-muted-foreground uppercase">Critical Risk Level</p>
+          </Card>
+          <Card className="p-6 bg-card border-border">
+            <TrendingUp className="w-8 h-8 text-info mb-3" />
+            <p className="text-2xl font-bold">
+              {Math.min(100, Math.round(
+                (sortedEvents.filter(e => e.domain === 'economical' || e.domain === 'market').length / Math.max(sortedEvents.length, 1)) * 100 * 3
+              ))}%
+            </p>
+            <p className="text-xs text-muted-foreground uppercase">Market Activity</p>
+          </Card>
+          <Card className="p-6 bg-card border-border">
+            <Building className="w-8 h-8 text-success mb-3" />
+            <p className="text-2xl font-bold">
+              {Math.round((opportunityEvents.length / Math.max(sortedEvents.length, 1)) * 100)}%
+            </p>
+            <p className="text-xs text-muted-foreground uppercase">Opportunity Index</p>
+          </Card>
+        </div>
+      )}
     </div>
   );
 };

main.py

@@ -90,14 +90,13 @@ def check_and_train_models():
             "name": "Stock Prediction",
             "check_paths": [
                 PROJECT_ROOT / "models" / "stock-price-prediction"
-                / "artifacts" / "models",
+                / "Artifacts",
             ],
-            "check_files": ["*.h5", "*.keras"],
+            "check_files": ["*.pkl", "*.h5", "*.keras"],
             "train_cmd": [
                 sys.executable,
                 str(PROJECT_ROOT / "models" / "stock-price-prediction"
-                    / "main.py"),
-                "--mode", "full"
+                    / "main.py")
             ]
         },
     ]
@@ -1710,9 +1709,29 @@ async def get_currency_prediction():
     predictor = get_currency_predictor()
 
     if predictor is None:
+        # Generate fallback prediction inline
+        import numpy as np
+        current_rate = 298.0
+        np.random.seed(int(datetime.now().timestamp()) % 2**31)
+        change_pct = np.random.normal(0.05, 0.3)
+        predicted_rate = current_rate * (1 + change_pct / 100)
+        
         return {
-            "status": "unavailable",
-            "message": "Currency prediction model not loaded"
+            "status": "success",
+            "prediction": {
+                "prediction_date": (datetime.now() + timedelta(days=1)).strftime("%Y-%m-%d"),
+                "generated_at": datetime.now().isoformat(),
+                "model_version": "fallback",
+                "is_fallback": True,
+                "current_rate": round(current_rate, 2),
+                "predicted_rate": round(predicted_rate, 2),
+                "expected_change": round(predicted_rate - current_rate, 2),
+                "expected_change_pct": round(change_pct, 3),
+                "direction": "strengthening" if change_pct < 0 else "weakening",
+                "direction_emoji": "📈" if change_pct < 0 else "📉",
+                "volatility_class": "low",
+                "note": "Using fallback - model initializing"
+            }
         }
 
     try:

models/stock-price-prediction/Artifacts/12_10_2025_03_04_56/data_validation/drift_report/report.yaml

@@ -21,4 +21,4 @@ StockName:
   p_value: 1.0
 Volume:
   drift_status: true
-  p_value: 3.715e-321
+  p_value: 9.305833669220032e-37

src/graphs/vectorizationAgentGraph.py

@@ -77,4 +77,3 @@ graph = VectorizationGraphBuilder(llm).build_graph()
 
 print("[OK] Vectorization Agent Graph compiled successfully")
 print("=" * 60 + "\n")
-

src/nodes/vectorizationAgentNode.py

@@ -389,7 +389,7 @@ class VectorizationAgentNode:
 
                 # For non-English languages, skip anomaly detection
                 # The ML model was trained on English embeddings only.
-                # Different BERT models (SinhalaBERTo, Tamil-BERT) have completely 
+                # Different BERT models (SinhalaBERTo, Tamil-BERT) have completely
                 # different embedding spaces - Tamil embeddings have magnitude ~0.64
                 # while English has ~7.5 and Sinhala ~13.7. They cannot be compared.
                 if language in ["sinhala", "tamil"]:
@@ -453,7 +453,7 @@ class VectorizationAgentNode:
     def run_trending_detection(self, state: VectorizationAgentState) -> Dict[str, Any]:
         """
         Step 2.6: Detect trending topics from the input texts.
-        
+
         Extracts key entities/topics and tracks their mention velocity.
         Identifies:
         - Trending topics (momentum > 2x normal)
@@ -461,9 +461,9 @@ class VectorizationAgentNode:
         - Topics with increasing momentum
         """
         logger.info("[VectorizationAgent] STEP 2.6: Trending Detection")
-        
+
         detection_results = state.get("language_detection_results", [])
-        
+
         if not detection_results:
             logger.warning("[VectorizationAgent] No texts for trending detection")
             return {
@@ -475,7 +475,7 @@ class VectorizationAgentNode:
                     "spike_alerts": [],
                 },
             }
-        
+
         # Import trending detector
         try:
             from src.utils.trending_detector import (
@@ -484,11 +484,12 @@ class VectorizationAgentNode:
                 get_trending_now,
                 get_spikes,
             )
+
             TRENDING_AVAILABLE = True
         except ImportError as e:
             logger.warning(f"[VectorizationAgent] Trending detector not available: {e}")
             TRENDING_AVAILABLE = False
-        
+
         if not TRENDING_AVAILABLE:
             return {
                 "current_step": "trending_detection",
@@ -499,19 +500,19 @@ class VectorizationAgentNode:
                     "spike_alerts": [],
                 },
             }
-        
+
         # Extract entities and record mentions
         entities_found = []
-        
+
         for item in detection_results:
             text = item.get("text", "")  # Field is 'text', not 'original_text'
             language = item.get("language", "english")
             post_id = item.get("post_id", "")
-            
+
             # Simple entity extraction (keywords, capitalized words, etc.)
             # In production, you'd use NER or more sophisticated extraction
             extracted = self._extract_entities(text, language)
-            
+
             for entity in extracted:
                 try:
                     # Record mention with trending detector
@@ -520,15 +521,17 @@ class VectorizationAgentNode:
                         source=entity.get("source", "feed"),
                         domain=entity.get("domain", "general"),
                     )
-                    entities_found.append({
-                        "entity": entity["text"],
-                        "type": entity.get("type", "keyword"),
-                        "post_id": post_id,
-                        "language": language,
-                    })
+                    entities_found.append(
+                        {
+                            "entity": entity["text"],
+                            "type": entity.get("type", "keyword"),
+                            "post_id": post_id,
+                            "language": language,
+                        }
+                    )
                 except Exception as e:
                     logger.debug(f"[VectorizationAgent] Failed to record mention: {e}")
-        
+
         # Get current trending topics and spikes
         try:
             trending_topics = get_trending_now(limit=10)
@@ -537,12 +540,12 @@ class VectorizationAgentNode:
             logger.warning(f"[VectorizationAgent] Failed to get trending data: {e}")
             trending_topics = []
             spike_alerts = []
-        
+
         logger.info(
             f"[VectorizationAgent] Trending detection: {len(entities_found)} entities, "
             f"{len(trending_topics)} trending, {len(spike_alerts)} spikes"
         )
-        
+
         return {
             "current_step": "trending_detection",
             "trending_results": {
@@ -553,65 +556,97 @@ class VectorizationAgentNode:
                 "spike_alerts": spike_alerts,
             },
         }
-    
-    def _extract_entities(self, text: str, language: str = "english") -> List[Dict[str, Any]]:
+
+    def _extract_entities(
+        self, text: str, language: str = "english"
+    ) -> List[Dict[str, Any]]:
         """
         Extract entities/topics from text for trending tracking.
-        
+
         Uses simple heuristics:
         - Capitalized words/phrases (potential proper nouns)
         - Hashtags
         - Common news keywords
-        
+
         In production, integrate with NER model for better extraction.
         """
         entities = []
-        
+
         if not text:
             return entities
-        
+
         import re
-        
+
         # Extract hashtags
-        hashtags = re.findall(r'#(\w+)', text)
+        hashtags = re.findall(r"#(\w+)", text)
         for tag in hashtags:
-            entities.append({
-                "text": tag.lower(),
-                "type": "hashtag",
-                "source": "hashtag",
-                "domain": "social",
-            })
-        
+            entities.append(
+                {
+                    "text": tag.lower(),
+                    "type": "hashtag",
+                    "source": "hashtag",
+                    "domain": "social",
+                }
+            )
+
         # Extract capitalized phrases (potential proper nouns)
         # Match 1-4 consecutive capitalized words
-        cap_phrases = re.findall(r'\b([A-Z][a-z]+(?: [A-Z][a-z]+){0,3})\b', text)
+        cap_phrases = re.findall(r"\b([A-Z][a-z]+(?: [A-Z][a-z]+){0,3})\b", text)
         for phrase in cap_phrases:
             # Skip common words
-            if phrase.lower() not in ['the', 'a', 'an', 'is', 'are', 'was', 'were', 'i', 'he', 'she', 'it']:
-                entities.append({
-                    "text": phrase,
-                    "type": "proper_noun",
-                    "source": "text",
-                    "domain": "general",
-                })
-        
+            if phrase.lower() not in [
+                "the",
+                "a",
+                "an",
+                "is",
+                "are",
+                "was",
+                "were",
+                "i",
+                "he",
+                "she",
+                "it",
+            ]:
+                entities.append(
+                    {
+                        "text": phrase,
+                        "type": "proper_noun",
+                        "source": "text",
+                        "domain": "general",
+                    }
+                )
+
         # News/event keywords
         news_keywords = [
-            'breaking', 'urgent', 'alert', 'emergency', 'crisis',
-            'earthquake', 'flood', 'tsunami', 'election', 'protest',
-            'strike', 'scandal', 'corruption', 'price', 'inflation',
+            "breaking",
+            "urgent",
+            "alert",
+            "emergency",
+            "crisis",
+            "earthquake",
+            "flood",
+            "tsunami",
+            "election",
+            "protest",
+            "strike",
+            "scandal",
+            "corruption",
+            "price",
+            "inflation",
         ]
-        
+
         text_lower = text.lower()
         for keyword in news_keywords:
             if keyword in text_lower:
-                entities.append({
-                    "text": keyword,
-                    "type": "news_keyword",
-                    "source": "keyword_match",
-                    "domain": "news",
-                })
-        
+                entities.append(
+                    {
+                        "text": keyword,
+                        "type": "news_keyword",
+                        "source": "keyword_match",
+                        "domain": "news",
+                    }
+                )
+
         # Deduplicate by text
         seen = set()
         unique_entities = []
@@ -620,9 +655,8 @@ class VectorizationAgentNode:
             if key not in seen:
                 seen.add(key)
                 unique_entities.append(e)
-        
-        return unique_entities[:15]  # Limit entities per text
 
+        return unique_entities[:15]  # Limit entities per text
 
     def generate_expert_summary(self, state: VectorizationAgentState) -> Dict[str, Any]:
         """