Update app.py

app.py

@@ -2,22 +2,35 @@ from fastapi import FastAPI
 from pydantic import BaseModel
 from llama_cpp import Llama
 from typing import List, Optional
+import httpx
+import os
 
 app = FastAPI()
 
 
-SYSTEM_PROMPT = """You are Edyx.
-You are a helpful, harmless, and honest AI assistant.
-"""
+GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
+GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions"
+GROQ_MODEL = "llama-3.3-70b-versatile"  
 
-llm = Llama(
-    model_path="/models/model.gguf",
 
-    n_ctx=4096,  
-    n_threads=2,
-    n_batch=128,
-    verbose=False
-)
+SYSTEM_PROMPT = """You are a helpful, harmless, and honest AI assistant.
+Provide clear and conversational responses."""
+
+
+local_llm = None
+
+def get_local_llm():
+    global local_llm
+    if local_llm is None:
+        print("Loading local fallback model...")
+        local_llm = Llama(
+            model_path="/models/model.gguf",
+            n_ctx=4096,
+            n_threads=2,
+            n_batch=128,
+            verbose=False
+        )
+    return local_llm
 
 class Message(BaseModel):
     role: str
@@ -25,40 +38,100 @@ class Message(BaseModel):
 
 class ChatRequest(BaseModel):
     messages: List[Message]
-
     max_tokens: Optional[int] = 1024
     temperature: Optional[float] = 0.7
     repetition_penalty: Optional[float] = 1.1
 
-@app.post("/v1/chat")
-def chat(req: ChatRequest):
-
+@app.get("/")
+def root():
+    return {"status": "edyx convo model running", "mode": "groq-primary"}
+
+async def call_groq_api(messages: List[Message], max_tokens: int, temperature: float):
+    """Try to get response from Groq API"""
+    if not GROQ_API_KEY:
+        raise Exception("GROQ_API_KEY not configured")
+    
+    groq_messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+    for m in messages:
+        groq_messages.append({"role": m.role, "content": m.content})
+    
+    async with httpx.AsyncClient(timeout=45.0) as client:
+        response = await client.post(
+            GROQ_API_URL,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {GROQ_API_KEY}"
+            },
+            json={
+                "model": GROQ_MODEL,
+                "messages": groq_messages,
+                "max_tokens": max_tokens,
+                "temperature": temperature
+            }
+        )
+        
+        if response.status_code != 200:
+            raise Exception(f"Groq API error: {response.status_code} - {response.text}")
+        
+        data = response.json()
+        return data["choices"][0]["message"]["content"], data["usage"]["total_tokens"]
+
+def call_local_model(messages: List[Message], max_tokens: int, temperature: float, repetition_penalty: float):
+    """Fallback to local llama model - YOUR ORIGINAL LOGIC"""
+    llm = get_local_llm()
+    
     prompt = SYSTEM_PROMPT + "\n\n"
-
-    for m in req.messages:
+    for m in messages:
         role = m.role.lower()
         if role == "system":
-
-             prompt = f"{m.content}\n\n"
+            prompt = f"{m.content}\n\n"
         else:
-             prompt += f"{role}: {m.content}\n"
-
+            prompt += f"{role}: {m.content}\n"
     prompt += "assistant:"
 
-
     output = llm(
         prompt,
-        max_tokens=req.max_tokens, 
-        temperature=req.temperature,
+        max_tokens=max_tokens,
+        temperature=temperature,
         top_p=0.9,
-        repeat_penalty=req.repetition_penalty,
+        repeat_penalty=repetition_penalty,
         stop=["user:", "assistant:", "<|end|>", "User:"]
     )
+    
+    return output["choices"][0]["text"].strip(), output["usage"]["total_tokens"]
 
-    text = output["choices"][0]["text"].strip()
-
-    return {
-        "model": "edyx-convo",
-        "text": text,
-        "tokens": output["usage"]["total_tokens"]
-    } 
+@app.post("/v1/chat")
+async def chat(req: ChatRequest):
+    # Try Groq API first (fast path)
+    try:
+        text, tokens = await call_groq_api(req.messages, req.max_tokens, req.temperature)
+        return {
+            "model": "edyx-convo",
+            "text": text,
+            "tokens": tokens,
+            "source": "primary"
+        }
+    except Exception as e:
+        print(f"Groq API failed: {e}, falling back to local model...")
+    
+    # Fallback to local model - YOUR ORIGINAL CODE
+    try:
+        text, tokens = call_local_model(
+            req.messages,
+            req.max_tokens,
+            req.temperature,
+            req.repetition_penalty
+        )
+        return {
+            "model": "edyx-convo",
+            "text": text,
+            "tokens": tokens,
+            "source": "fallback"
+        }
+    except Exception as e:
+        return {
+            "model": "edyx-convo",
+            "text": f"Error: Both primary and fallback failed. {str(e)}",
+            "tokens": 0,
+            "source": "error"
+        }