To space

backend/api/llm_evaluator.py

@@ -1,14 +1,23 @@
 """
-LLM Evaluator – analyzes BESS-RL evaluation results using Gemini or Hugging Face.
+LLM Evaluator – analyzes BESS-RL evaluation results using the Hugging Face Router.
 Includes a heuristic fallback for when API quotas are exceeded.
+All models (Gemma, Qwen, GPT) are routed via the Hugging Face OpenAI-compatible API.
 """
 import os
 import json
 import requests
+import openai
 
-GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-1.5-flash")
-HF_MODEL = os.getenv("HF_MODEL", "google/gemma-2-9b-it")
-LLM_PROVIDER = os.getenv("LLM_PROVIDER", "GEMINI").upper()
+# Default Model IDs for the HF Router
+DEFAULT_GEMMA_MODEL = os.getenv("HF_GEMMA_MODEL", "google/gemma-4-31b-it")
+DEFAULT_QWEN_MODEL = os.getenv("HF_QWEN_MODEL", "Qwen/Qwen2.5-72B-Instruct")
+DEFAULT_GPT_MODEL = os.getenv("HF_GPT_MODEL", "openai/gpt-4o-mini")
+
+TASK_DESCRIPTIONS = {
+    "easy":   "Energy Arbitrage only (buy cheap, sell expensive)",
+    "medium": "Energy Arbitrage + Frequency Regulation",
+    "hard":   "Energy Arbitrage + Frequency Regulation + Peak Shaving",
+}
 
 _FALLBACK_BASE = {
     "available": False,
@@ -25,27 +34,18 @@ _FALLBACK_BASE = {
     "provider": "UNKNOWN"
 }
 
-TASK_DESCRIPTIONS = {
-    "easy":   "Energy Arbitrage only (buy cheap, sell expensive)",
-    "medium": "Energy Arbitrage + Frequency Regulation",
-    "hard":   "Energy Arbitrage + Frequency Regulation + Peak Shaving",
-}
-
 def _get_heuristic_analysis(data: dict) -> dict:
     """Provides a rule-based assessment when LLMs are unavailable."""
     scores = data.get("scores", {})
     overall = scores.get("overall", 0)
     task = data.get("task", "hard")
     
-    # Determine Verdict
     if overall >= 0.85: verdict = "Excellent"
     elif overall >= 0.70: verdict = "Good"
     elif overall >= 0.50: verdict = "Needs Improvement"
     else: verdict = "Poor"
     
     strengths, weaknesses, recommendations = [], [], []
-    
-    # Analysis logic
     if scores.get("reward", 0) > 0.8: strengths.append("Strong reward optimization")
     else: weaknesses.append("Sub-optimal profit generation")
     
@@ -60,8 +60,6 @@ def _get_heuristic_analysis(data: dict) -> dict:
         weaknesses.append("Aggressive battery cycling")
         recommendations.append("Increase degradation cost coefficient to preserve battery health")
 
-    # Calculate an independent heuristic score (weighted average of components)
-    # This makes the heuristic score distinct from the simulator's engine score
     h_score = (
         scores.get("reward", 0) * 0.4 +
         scores.get("soc_readiness", 0) * 0.2 +
@@ -126,101 +124,56 @@ Cycle Discipline     : {scores.get('cycle_discipline', 0):.3f}
 }}
 """
 
-def _get_gemini_analysis(data: dict) -> dict:
-    api_key = os.getenv("GEMINI_API_KEY") or os.getenv("GEMINI_TOKEN")
-    if not api_key: raise ValueError("GEMINI_API_KEY not found in Space secrets.")
+def _get_router_analysis(data: dict, model_id: str) -> dict:
+    """Calls the Hugging Face Router (OpenAI-compatible) for LLM analysis."""
+    api_token = os.getenv("HF_TOKEN") or os.getenv("PowerGrid") or os.getenv("HUGGING_FACE_HUB_TOKEN")
+    if not api_token:
+        raise ValueError("Hugging Face token not found (expected HF_TOKEN).")
     
-    from google import genai
-    from google.genai import types
-    client = genai.Client(api_key=api_key)
-    prompt = _build_prompt(data)
-    response = client.models.generate_content(
-        model=GEMINI_MODEL,
-        contents=prompt,
-        config=types.GenerateContentConfig(
-            temperature=0.3,
-        ),
+    # Initialize OpenAI client pointing to HF Router
+    client = openai.OpenAI(
+        base_url="https://router.huggingface.co/v1",
+        api_key=api_token
     )
-    # Extract JSON string safely
-    text = response.text.strip()
-    if "```json" in text:
-        text = text.split("```json")[1].split("```")[0]
-    elif "```" in text:
-        text = text.split("```")[1].split("```")[0]
-        
-    return {**json.loads(text.strip()), "available": True, "provider": "GEMINI"}
-
-def _get_openai_analysis(data: dict, model_name: str) -> dict:
-    api_key = os.getenv("OPENAI_API_KEY") or os.getenv("OPENAI_TOKEN")
-    if not api_key: raise ValueError("OPENAI_API_KEY not found in Space secrets.")
     
-    import openai
-    client = openai.OpenAI(api_key=api_key)
     prompt = _build_prompt(data)
     
     response = client.chat.completions.create(
-        model=model_name or "gpt-4o",
+        model=model_id,
         messages=[{"role": "user", "content": prompt}],
         temperature=0.3,
         response_format={"type": "json_object"}
     )
-    return {**json.loads(response.choices[0].message.content), "available": True, "provider": "OPENAI"}
-
-def _get_hf_analysis(data: dict, model_name: str) -> dict:
-    api_token = os.getenv("HF_TOKEN") or os.getenv("PowerGrid") or os.getenv("HUGGING_FACE_HUB_TOKEN")
-    if not api_token: raise ValueError("Hugging Face token not found (expected HF_TOKEN or PowerGrid secret).")
     
-    model = model_name or HF_MODEL
-    api_url = f"https://api-inference.huggingface.co/models/{model}"
-    headers = {"Authorization": f"Bearer {api_token}"}
-    prompt = _build_prompt(data) + "\nJSON Output:"
-    
-    response = requests.post(api_url, headers=headers, json={
-        "inputs": prompt,
-        "parameters": {"max_new_tokens": 1000, "return_full_text": False}
-    })
-    
-    if response.status_code != 200:
-        raise ValueError(f"HF API Error: {response.text}")
-        
-    # Extract JSON string from response
-    try:
-        res_json = response.json()
-        if isinstance(res_json, list) and len(res_json) > 0:
-            text = res_json[0].get('generated_text', '')
-        else:
-            text = str(res_json)
-            
-        if "```json" in text:
-            text = text.split("```json")[1].split("```")[0]
-        elif "```" in text:
-            text = text.split("```")[1].split("```")[0]
-            
-        return {**json.loads(text.strip()), "available": True, "provider": "HF"}
-    except:
-        raise ValueError("Failed to parse JSON from Hugging Face model response")
+    content = response.choices[0].message.content
+    return {**json.loads(content), "available": True, "provider": f"HF:{model_id}"}
 
 def get_llm_analysis(data: dict, provider: str = "GEMINI", model_name: str = None) -> dict:
-    """Main entry point for LLM analysis with explicit provider support."""
+    """Main entry point for LLM analysis. Routes all providers through HF Router."""
     try:
+        # Map provider to specific model ID on HF Router
         if provider == "GEMINI":
-            return _get_gemini_analysis(data)
+            # Using Gemma 4 as the replacement for Gemini
+            model_id = model_name or DEFAULT_GEMMA_MODEL
         elif provider == "QWEN":
-            return _get_hf_analysis(data, model_name or "Qwen/Qwen2.5-72B-Instruct")
+            model_id = model_name or DEFAULT_QWEN_MODEL
         elif provider == "OPENAI":
-            return _get_openai_analysis(data, model_name or "gpt-4o")
+            # Routing OpenAI requests through HF Router as requested
+            model_id = model_name or DEFAULT_GPT_MODEL
         elif provider == "HF":
-            return _get_hf_analysis(data, model_name)
+            model_id = model_name or DEFAULT_GEMMA_MODEL
+        else:
+            # Fallback for unknown providers
+            model_id = model_name or DEFAULT_GEMMA_MODEL
+            
+        return _get_router_analysis(data, model_id)
+        
     except Exception as e:
         error_msg = str(e)
-        # If the user specifically requested a model and it failed, tell them why.
         return {
             **_FALLBACK_BASE,
             "available": False,
             "summary": f"Evaluation Failed: {error_msg}",
             "error": error_msg,
-            "detailed_analysis": f"The requested {provider} analysis could not be completed. \n\nReason: {error_msg}\n\nPlease check your Hugging Face Space secrets settings."
+            "detailed_analysis": f"The requested {provider} analysis could not be completed using model {model_id if 'model_id' in locals() else 'unknown'}.\n\nReason: {error_msg}\n\nPlease check your Hugging Face Space secrets and token permissions."
         }
-
-    # Global fallback logic (only if provider was None/Unknown, which shouldn't happen now)
-    return _get_heuristic_analysis(data)