update model embedding, fix dockerfile

Dockerfile

@@ -2,14 +2,15 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Cài đặt các gói hệ thống cần thiết
+# Cài đặt các gói hệ thống cần thiết (tối giản, tương thích Debian trixie)
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
-    software-properties-common \
     git \
     gcc \
     python3-dev \
+    libpq-dev \
+    pkg-config \
     && rm -rf /var/lib/apt/lists/*
 
 # Sao chép các file yêu cầu trước để tận dụng cache của Docker

README.md

@@ -190,7 +190,9 @@ PINECONE_API_KEY=your-pinecone-api-key
 PINECONE_INDEX_NAME=your-pinecone-index-name
 PINECONE_ENVIRONMENT=gcp-starter
 
-# Google Gemini API key
+# Google Gemini API key (REQUIRED for embeddings)
+# Get your API key from: https://makersuite.google.com/app/apikey
+# Note: This project uses Gemini embedding-001 model which has free tier limits
 GOOGLE_API_KEY=your-google-api-key
 
 # WebSocket configuration

app/api/mongodb_routes.py

@@ -167,6 +167,10 @@ async def get_history(user_id: str, n: int = Query(3, ge=1, le=10)):
     
     - **user_id**: User's ID from Telegram
     - **n**: Number of most recent interactions to return (default: 3, min: 1, max: 10)
+    
+    Returns a formatted string with conversation history in the format:
+    User: [question]
+    Bot: [answer]
     """
     try:
         # Kiểm tra kết nối MongoDB

app/api/rag_routes.py

@@ -58,7 +58,8 @@ logger = logging.getLogger(__name__)
 
 # Configure Google Gemini API
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
-genai.configure(api_key=GOOGLE_API_KEY)
+if GOOGLE_API_KEY:
+    genai.configure(api_key=GOOGLE_API_KEY)
 KEYWORD_LIST = os.getenv("KEYWORDS")
 
 # Create router
@@ -159,8 +160,19 @@ Your response:
 async def get_embedding(text: str):
     """Get embedding from Google Gemini API"""
     try:
-        # Initialize embedding model
-        embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        # Check if Google API key is available
+        if not GOOGLE_API_KEY:
+            raise HTTPException(status_code=500, detail="Google API key not configured")
+        
+        # Log API key for debugging (masked)
+        masked_key = GOOGLE_API_KEY[:8] + "..." + GOOGLE_API_KEY[-4:] if len(GOOGLE_API_KEY) > 12 else "***"
+        logger.info(f"Using Google API key for embedding: {masked_key}")
+        
+        # Initialize embedding model (using latest model)
+        embedding_model = GoogleGenerativeAIEmbeddings(
+            model="models/text-embedding-004",  # Updated to latest model with correct prefix
+            google_api_key=GOOGLE_API_KEY
+        )
         
         # Generate embedding
         result = await embedding_model.aembed_query(text)
@@ -169,10 +181,18 @@ async def get_embedding(text: str):
         return {
             "embedding": result,
             "text": text,
-            "model": "embedding-001"
+            "model": "models/text-embedding-004"
         }
     except Exception as e:
         logger.error(f"Error generating embedding: {e}")
+        
+        # Check for quota exceeded error
+        if "quota" in str(e).lower() or "429" in str(e):
+            raise HTTPException(
+                status_code=429, 
+                detail="Google API quota exceeded. Please check your billing or wait for quota reset."
+            )
+        
         raise HTTPException(status_code=500, detail=f"Failed to generate embedding: {str(e)}")
 
 # Endpoint for generating embeddings

app/database/pinecone.py

@@ -410,8 +410,25 @@ class ThresholdRetriever(BaseRetriever):
             embedding = self.embeddings.embed_query(query)
         except Exception as e:
             logger.error(f"Error generating embedding: {e}")
+            
+            # Check for quota exceeded error
+            if "quota" in str(e).lower() or "429" in str(e):
+                logger.error("Google API quota exceeded. Please check your billing or wait for quota reset.")
+                return None
+                
             # Fallback to creating a new embedding model if needed
-            embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+            if not GOOGLE_API_KEY:
+                logger.error("Google API key not configured for fallback embedding")
+                return None
+                
+            # Log API key for debugging (masked)
+            masked_key = GOOGLE_API_KEY[:8] + "..." + GOOGLE_API_KEY[-4:] if len(GOOGLE_API_KEY) > 12 else "***"
+            logger.info(f"Using Google API key for fallback: {masked_key}")
+                
+            embedding_model = GoogleGenerativeAIEmbeddings(
+                model="models/text-embedding-004",  # Updated to latest model with correct prefix
+                google_api_key=GOOGLE_API_KEY
+            )
             embedding = embedding_model.embed_query(query)
         
         # Perform search with advanced options - avoid asyncio.run()
@@ -523,8 +540,20 @@ def get_chain(
         start_time = time.time()
         logger.info("Initializing new retriever chain with threshold-based filtering")
         
-        # Initialize embeddings model
-        embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        # Check if Google API key is available
+        if not GOOGLE_API_KEY:
+            logger.error("Google API key not configured")
+            return None
+            
+        # Log API key for debugging (masked)
+        masked_key = GOOGLE_API_KEY[:8] + "..." + GOOGLE_API_KEY[-4:] if len(GOOGLE_API_KEY) > 12 else "***"
+        logger.info(f"Using Google API key: {masked_key}")
+            
+        # Initialize embeddings model (using latest model)
+        embeddings = GoogleGenerativeAIEmbeddings(
+            model="models/text-embedding-004",  # Updated to latest model with correct prefix
+            google_api_key=GOOGLE_API_KEY
+        )
         
         # Get index
         pinecone_index = get_pinecone_index()

app/models/mongodb_models.py

@@ -52,4 +52,4 @@ class QuestionAnswer(BaseModel):
 
 class HistoryResponse(BaseModel):
     """Response model for history"""
-    history: List[QuestionAnswer] 
+    history: str 

app/utils/pdf_processor.py

@@ -99,6 +99,10 @@ class PDFProcessor:
             if not self.google_api_key:
                 raise ValueError("Google API key not found in environment variables")
             
+            # Log API key for debugging (masked)
+            masked_key = self.google_api_key[:8] + "..." + self.google_api_key[-4:] if len(self.google_api_key) > 12 else "***"
+            logger.info(f"[{self.correlation_id}] Using Google API key: {masked_key}")
+            
             genai.configure(api_key=self.google_api_key)
             
             # First, get the expected dimensions from Pinecone
@@ -110,9 +114,9 @@ class PDFProcessor:
             pinecone_dimension = stats.dimension
             logger.info(f"[{self.correlation_id}] Pinecone index dimension: {pinecone_dimension}")
             
-            # Create embedding model
+            # Create embedding model (using latest model)
             embedding_model = GoogleGenerativeAIEmbeddings(
-                model="models/embedding-001",
+                model="models/text-embedding-004",  # Updated to latest model with correct prefix
                 google_api_key=self.google_api_key,
                 task_type="retrieval_document"  # Use document embedding mode for longer text
             )