Updated app.py with dynamic API call

app.py

@@ -10,9 +10,10 @@ import os
 from contextlib import asynccontextmanager
 import tempfile
 import shutil
+import random
 
 # Third-party imports
-from openai import AsyncOpenAI
+from openai import OpenAI, AsyncOpenAI
 from qdrant_client import AsyncQdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue
 from sentence_transformers import SentenceTransformer
@@ -27,12 +28,13 @@ class Message(BaseModel):
     content: str = Field(..., description="The content of the message")
 
 class ChatCompletionRequest(BaseModel):
-    model: str = Field(default="mixtral-8x7b-32768", description="Model to use")
+    model: str = Field(default="auto", description="Model to use (auto for dynamic selection)")
     messages: List[Message] = Field(..., description="List of messages")
     max_tokens: Optional[int] = Field(default=1024, description="Maximum tokens to generate")
     temperature: Optional[float] = Field(default=0.7, description="Temperature for sampling")
     stream: Optional[bool] = Field(default=False, description="Whether to stream responses")
     top_p: Optional[float] = Field(default=1.0, description="Top-p sampling parameter")
+    provider: Optional[str] = Field(default="random", description="Provider to use (random, openrouter, groq)")
 
 class ChatCompletionResponse(BaseModel):
     id: str
@@ -59,20 +61,152 @@ class DocumentSearchRequest(BaseModel):
 
 # Configuration
 class Config:
+    # Provider API Keys
+    OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
     GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-    GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1")
+    
+    # Vector DB Configuration
     QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
     QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
     COLLECTION_NAME = os.getenv("COLLECTION_NAME", "documents")
+    
+    # Embedding Configuration
     EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
-    TOP_K = int(os.getenv("TOP_K", "10"))  # Increased from 5
-    SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", "0.1"))  # Lowered from 0.7
+    TOP_K = int(os.getenv("TOP_K", "10"))
+    SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", "0.1"))
     DEVICE = os.getenv("DEVICE", "cuda" if torch.cuda.is_available() else "cpu")
 
+# Models
+OPENROUTER_MODELS = ["deepseek/deepseek-chat-v3-0324:free", "deepseek/deepseek-r1-0528:free", "qwen/qwen3-235b-a22b:free", "google/gemini-2.0-flash-exp:free"]
+GROQ_MODELS = ["llama-3.3-70b-versatile"]
+
+class DynamicOpenAIService:
+    """Service for dynamic OpenAI provider selection"""
+    
+    def __init__(self):
+        self.validate_api_keys()
+    
+    def validate_api_keys(self):
+        """Validate that at least one API key is available"""
+        if not Config.OPENROUTER_API_KEY and not Config.GROQ_API_KEY:
+            raise ValueError("At least one API key (OPENROUTER_API_KEY or GROQ_API_KEY) must be provided")
+        
+        if not Config.OPENROUTER_API_KEY:
+            print("Warning: OPENROUTER_API_KEY not found, will only use Groq")
+        if not Config.GROQ_API_KEY:
+            print("Warning: GROQ_API_KEY not found, will only use OpenRouter")
+    
+    def get_client(self, provider="random"):
+        """Get OpenAI client for specified provider"""
+        available_providers = []
+        
+        if Config.OPENROUTER_API_KEY:
+            available_providers.append("openrouter")
+        if Config.GROQ_API_KEY:
+            available_providers.append("groq")
+        
+        if not available_providers:
+            raise ValueError("No API keys available for any provider")
+        
+        if provider == "random":
+            provider = random.choice(available_providers)
+        elif provider not in available_providers:
+            # Fallback to available provider
+            provider = available_providers[0]
+            print(f"Requested provider not available, using {provider}")
+        
+        print(f"Selected provider: {provider}")
+        
+        if provider == "openrouter":
+            return (
+                OpenAI(api_key=Config.OPENROUTER_API_KEY, base_url="https://openrouter.ai/api/v1"), 
+                OPENROUTER_MODELS,
+                provider
+            )
+        else:  # groq
+            return (
+                OpenAI(api_key=Config.GROQ_API_KEY, base_url="https://api.groq.com/openai/v1"), 
+                GROQ_MODELS,
+                provider
+            )
+    
+    async def get_async_client(self, provider="random"):
+        """Get AsyncOpenAI client for specified provider"""
+        available_providers = []
+        
+        if Config.OPENROUTER_API_KEY:
+            available_providers.append("openrouter")
+        if Config.GROQ_API_KEY:
+            available_providers.append("groq")
+        
+        if not available_providers:
+            raise ValueError("No API keys available for any provider")
+        
+        if provider == "random":
+            provider = random.choice(available_providers)
+        elif provider not in available_providers:
+            # Fallback to available provider
+            provider = available_providers[0]
+            print(f"Requested provider not available, using {provider}")
+        
+        print(f"Selected provider: {provider}")
+        
+        if provider == "openrouter":
+            return (
+                AsyncOpenAI(api_key=Config.OPENROUTER_API_KEY, base_url="https://openrouter.ai/api/v1"), 
+                OPENROUTER_MODELS,
+                provider
+            )
+        else:  # groq
+            return (
+                AsyncOpenAI(api_key=Config.GROQ_API_KEY, base_url="https://api.groq.com/openai/v1"), 
+                GROQ_MODELS,
+                provider
+            )
+    
+    def get_text_response(self, prompt, provider="random", model=None):
+        """Get text response from AI"""
+        client, models, selected_provider = self.get_client(provider)
+        
+        if not model or model == "auto":
+            model = random.choice(models)
+        
+        print(f"Using model: {model}")
+        
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1024,
+            temperature=0.7
+        )
+        
+        return response.choices[0].message.content
+    
+    def get_text_response_streaming(self, prompt, provider="random", model=None):
+        """Get streaming text response from AI"""
+        client, models, selected_provider = self.get_client(provider)
+        
+        if not model or model == "auto":
+            model = random.choice(models)
+        
+        print(f"Using model: {model}")
+        
+        stream = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1024,
+            temperature=0.7,
+            stream=True
+        )
+        
+        for chunk in stream:
+            if chunk.choices[0].delta.content is not None:
+                yield chunk.choices[0].delta.content
+
 class ApplicationState:
     """Application state container"""
     def __init__(self):
-        self.openai_client = None
+        self.openai_service = None
         self.qdrant_client = None
         self.embedding_service = None
         self.document_manager = None
@@ -421,7 +555,6 @@ class RAGService:
                 return []
             
             # Use a lower similarity threshold for RAG to get more results
-            # Try multiple thresholds if needed
             min_score = 0.1  # Lower threshold for RAG
             
             print(f"RAG Search - Query: '{query}', Limit: {top_k}, Min Score: {min_score}")
@@ -457,7 +590,7 @@ class RAGService:
         if not results:
             return query
         
-        # Build context parts like in your example
+        # Build context parts
         context_parts = []
         for result in results:
             context_parts.append(f"Source: {result['file_path']}\n{result['text']}")
@@ -478,25 +611,14 @@ Answer the question directly and naturally. Also Respond it in Markdown Format""
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # Startup
-    if not Config.GROQ_API_KEY:
-        raise ValueError("GROQ_API_KEY environment variable is required")
-    
     print("Initializing services...")
     
-    # Initialize OpenAI client with Groq endpoint
+    # Initialize dynamic OpenAI service
     try:
-        print(f"Configuring OpenAI client with:")
-        print(f"  Base URL: {Config.GROQ_BASE_URL}")
-        print(f"  API Key: {'*' * 10}...{Config.GROQ_API_KEY[-4:] if Config.GROQ_API_KEY else 'None'}")
-        
-        app_state.openai_client = AsyncOpenAI(
-            api_key=Config.GROQ_API_KEY,
-            base_url=Config.GROQ_BASE_URL,
-            timeout=60.0
-        )
-        print("✓ OpenAI client initialized with Groq endpoint")
+        app_state.openai_service = DynamicOpenAIService()
+        print("✓ Dynamic OpenAI service initialized")
     except Exception as e:
-        print(f"✗ Error initializing OpenAI client: {e}")
+        print(f"✗ Error initializing OpenAI service: {e}")
         raise e
     
     # Initialize Qdrant client
@@ -541,9 +663,6 @@ async def lifespan(app: FastAPI):
     if app_state.qdrant_client:
         await app_state.qdrant_client.close()
         print("✓ Qdrant client closed")
-    if app_state.openai_client:
-        await app_state.openai_client.close()
-        print("✓ OpenAI client closed")
     if app_state.embedding_service and hasattr(app_state.embedding_service, 'executor'):
         app_state.embedding_service.executor.shutdown(wait=True)
         print("✓ Embedding service executor shutdown")
@@ -554,15 +673,15 @@ async def lifespan(app: FastAPI):
 
 # Initialize FastAPI app
 app = FastAPI(
-    title="Enhanced RAG API with Document Management",
-    description="OpenAI-compatible API for RAG with document management using Groq and Qdrant",
+    title="Enhanced RAG API with Dynamic Provider Selection",
+    description="OpenAI-compatible API for RAG with dynamic provider selection (OpenRouter/Groq) and document management",
     version="1.0.0",
     lifespan=lifespan
 )
 
 @app.get("/")
 async def root():
-    return {"message": "Enhanced RAG API with Document Management", "status": "running"}
+    return {"message": "Enhanced RAG API with Dynamic Provider Selection", "status": "running"}
 
 @app.get("/health")
 async def health_check():
@@ -586,37 +705,61 @@ async def health_check():
         except Exception as e:
             embedding_health = {"status": "error", "error": str(e)}
     
-    # Test OpenAI client
-    if app_state.openai_client is None:
-        openai_health = {"status": "not_initialized", "error": "OpenAI client is None"}
+    # Test OpenAI service
+    if app_state.openai_service is None:
+        openai_health = {"status": "not_initialized", "error": "OpenAI service is None"}
     else:
         try:
-            test_response = await app_state.openai_client.chat.completions.create(
-                model="mixtral-8x7b-32768",
-                messages=[{"role": "user", "content": "test"}],
-                max_tokens=1
-            )
-            openai_health = {"status": "healthy", "test_response_id": test_response.id}
+            # Test both providers if available
+            test_results = {}
+            if Config.OPENROUTER_API_KEY:
+                try:
+                    client, models, provider = app_state.openai_service.get_client("openrouter")
+                    test_response = client.chat.completions.create(
+                        model=models[0],
+                        messages=[{"role": "user", "content": "test"}],
+                        max_tokens=1
+                    )
+                    test_results["openrouter"] = {"status": "healthy", "model": models[0]}
+                except Exception as e:
+                    test_results["openrouter"] = {"status": "error", "error": str(e)}
+            
+            if Config.GROQ_API_KEY:
+                try:
+                    client, models, provider = app_state.openai_service.get_client("groq")
+                    test_response = client.chat.completions.create(
+                        model=models[0],
+                        messages=[{"role": "user", "content": "test"}],
+                        max_tokens=1
+                    )
+                    test_results["groq"] = {"status": "healthy", "model": models[0]}
+                except Exception as e:
+                    test_results["groq"] = {"status": "error", "error": str(e)}
+            
+            openai_health = {"status": "healthy", "providers": test_results}
         except Exception as e:
             openai_health = {"status": "error", "error": str(e)}
     
     return {
         "status": "healthy" if app_state.embedding_service is not None else "unhealthy",
-        "openai_client": openai_health,
+        "openai_service": openai_health,
         "qdrant": qdrant_status,
         "embedding_service": embedding_health,
         "document_manager": "initialized" if app_state.document_manager else "not_initialized",
         "collection": Config.COLLECTION_NAME,
         "embedding_model": Config.EMBEDDING_MODEL,
-        "groq_endpoint": Config.GROQ_BASE_URL
+        "available_providers": {
+            "openrouter": bool(Config.OPENROUTER_API_KEY),
+            "groq": bool(Config.GROQ_API_KEY)
+        }
     }
 
 @app.post("/v1/chat/completions")
 async def chat_completions(request: ChatCompletionRequest):
-    """OpenAI-compatible chat completions endpoint with enhanced RAG"""
+    """OpenAI-compatible chat completions endpoint with enhanced RAG and dynamic provider selection"""
     
-    if not app_state.openai_client:
-        raise HTTPException(status_code=500, detail="OpenAI client not initialized")
+    if not app_state.openai_service:
+        raise HTTPException(status_code=500, detail="OpenAI service not initialized")
     
     try:
         # Get the last user message for retrieval
@@ -665,10 +808,21 @@ async def chat_completions(request: ChatCompletionRequest):
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
 async def create_chat_completion(messages: List[Dict], request: ChatCompletionRequest) -> ChatCompletionResponse:
-    """Create a non-streaming chat completion"""
+    """Create a non-streaming chat completion using dynamic provider selection"""
     try:
-        response = await app_state.openai_client.chat.completions.create(
-            model=request.model,
+        # Get async client with dynamic provider selection
+        client, models, selected_provider = await app_state.openai_service.get_async_client(request.provider)
+        
+        # Select model
+        if request.model == "auto" or not request.model:
+            selected_model = random.choice(models)
+        else:
+            selected_model = request.model
+        
+        print(f"Using provider: {selected_provider}, model: {selected_model}")
+        
+        response = await client.chat.completions.create(
+            model=selected_model,
             messages=messages,
             max_tokens=request.max_tokens,
             temperature=request.temperature,
@@ -679,7 +833,7 @@ async def create_chat_completion(messages: List[Dict], request: ChatCompletionRe
         result = ChatCompletionResponse(
             id=response.id,
             created=response.created,
-            model=response.model,
+            model=f"{selected_provider}:{response.model}",  # Include provider in model name
             choices=[{
                 "index": choice.index,
                 "message": {
@@ -702,10 +856,21 @@ async def create_chat_completion(messages: List[Dict], request: ChatCompletionRe
         raise HTTPException(status_code=500, detail=f"Error calling OpenAI API: {str(e)}")
 
 async def stream_chat_completion(messages: List[Dict], request: ChatCompletionRequest) -> AsyncGenerator[str, None]:
-    """Stream chat completion responses"""
+    """Stream chat completion responses using dynamic provider selection"""
     try:
-        stream = await app_state.openai_client.chat.completions.create(
-            model=request.model,
+        # Get async client with dynamic provider selection
+        client, models, selected_provider = await app_state.openai_service.get_async_client(request.provider)
+        
+        # Select model
+        if request.model == "auto" or not request.model:
+            selected_model = random.choice(models)
+        else:
+            selected_model = request.model
+        
+        print(f"Using provider: {selected_provider}, model: {selected_model}")
+        
+        stream = await client.chat.completions.create(
+            model=selected_model,
             messages=messages,
             max_tokens=request.max_tokens,
             temperature=request.temperature,
@@ -720,7 +885,7 @@ async def stream_chat_completion(messages: List[Dict], request: ChatCompletionRe
                     chunk_response = ChatCompletionChunk(
                         id=chunk.id,
                         created=chunk.created,
-                        model=chunk.model,
+                        model=f"{selected_provider}:{chunk.model}",  # Include provider in model name
                         choices=[{
                             "index": choice.index,
                             "delta": {
@@ -912,6 +1077,46 @@ async def get_collection_info():
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error getting collection info: {str(e)}")
 
+# New endpoint to get available providers and models
+@app.get("/v1/providers")
+async def get_providers():
+    """Get available providers and their models"""
+    try:
+        if not app_state.openai_service:
+            raise HTTPException(status_code=500, detail="OpenAI service not initialized")
+        
+        available_providers = {}
+        
+        if Config.OPENROUTER_API_KEY:
+            available_providers["openrouter"] = {
+                "status": "available",
+                "models": OPENROUTER_MODELS
+            }
+        else:
+            available_providers["openrouter"] = {
+                "status": "unavailable",
+                "reason": "API key not provided"
+            }
+        
+        if Config.GROQ_API_KEY:
+            available_providers["groq"] = {
+                "status": "available", 
+                "models": GROQ_MODELS
+            }
+        else:
+            available_providers["groq"] = {
+                "status": "unavailable",
+                "reason": "API key not provided"
+            }
+        
+        return {
+            "providers": available_providers,
+            "default_selection": "random"
+        }
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error getting providers: {str(e)}")
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)