clear history and fix bug websocket connection

.dockerignore

@@ -0,0 +1,46 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# Environment files
+.env
+.env.*
+!.env.example
+
+# Python cache files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+.pytest_cache/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Logs
+*.log
+
+# Tests
+tests/
+
+# Docker related
+Dockerfile
+docker-compose.yml
+.dockerignore
+
+# Other files
+.vscode/
+.idea/
+*.swp
+*.swo
+.DS_Store
+.coverage
+htmlcov/
+.mypy_cache/
+.tox/
+.nox/
+instance/
+.webassets-cache
+main.py 

.env.example

@@ -0,0 +1,33 @@
+# PostgreSQL Configuration
+DB_CONNECTION_MODE=aiven
+AIVEN_DB_URL=postgresql://username:password@host:port/dbname?sslmode=require
+
+# MongoDB Configuration
+MONGODB_URL=mongodb+srv://username:password@cluster.mongodb.net/?retryWrites=true&w=majority
+DB_NAME=Telegram
+COLLECTION_NAME=session_chat
+
+# Pinecone configuration
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_INDEX_NAME=your-pinecone-index-name
+PINECONE_ENVIRONMENT=gcp-starter
+
+# Google Gemini API key
+GOOGLE_API_KEY=your-google-api-key
+
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+
+# Application settings
+ENVIRONMENT=production
+DEBUG=false
+PORT=7860 
+
+# Cache Configuration
+CACHE_TTL_SECONDS=300
+CACHE_CLEANUP_INTERVAL=60
+CACHE_MAX_SIZE=1000
+HISTORY_QUEUE_SIZE=10
+HISTORY_CACHE_TTL=3600 

.gitignore

@@ -0,0 +1,83 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+.cache/
+coverage.xml
+*.cover
+.mypy_cache/
+
+# Environment
+.env
+.venv
+env/
+venv/
+ENV/
+
+# VSCode
+.vscode/
+*.code-workspace
+.history/
+
+# PyCharm
+.idea/
+*.iml
+*.iws
+*.ipr
+*.iws
+out/
+.idea_modules/
+
+# Logs and databases
+*.log
+*.sql
+*.sqlite
+*.db
+
+# Tests
+tests/
+
+Admin_bot/
+
+Pix-Agent/
+
+# Hugging Face Spaces
+.gitattributes
+
+# OS specific
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+Icon?
+ehthumbs.db
+Thumbs.db
+
+# Project specific
+*.log
+.env
+main.py
+
+test/

Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Cài đặt các gói hệ thống cần thiết
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    software-properties-common \
+    git \
+    gcc \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Sao chép các file yêu cầu trước để tận dụng cache của Docker
+COPY requirements.txt .
+
+# Cài đặt các gói Python
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Ensure langchain-core is installed
+RUN pip install --no-cache-dir langchain-core==0.1.19
+
+# Sao chép toàn bộ code vào container
+COPY . .
+
+# Mở cổng mà ứng dụng sẽ chạy
+EXPOSE 7860
+
+# Chạy ứng dụng với uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"] 

README.md

@@ -0,0 +1,419 @@
+---
+title: PIX Project Backend
+emoji: 🤖
+colorFrom: blue
+colorTo: green
+sdk: docker
+sdk_version: "3.0.0"
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# PIX Project Backend
+
+[![FastAPI](https://img.shields.io/badge/FastAPI-0.103.1-009688?style=flat&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/)
+[![Python 3.11](https://img.shields.io/badge/Python-3.11-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/)
+[![HuggingFace Spaces](https://img.shields.io/badge/HuggingFace-Spaces-yellow?style=flat&logo=huggingface&logoColor=white)](https://huggingface.co/spaces)
+
+Backend API for PIX Project with MongoDB, PostgreSQL and RAG integration. This project provides a comprehensive backend solution for managing FAQ items, emergency contacts, events, and a RAG-based question answering system.
+
+## Features
+
+- **MongoDB Integration**: Store user sessions and conversation history
+- **PostgreSQL Integration**: Manage FAQ items, emergency contacts, and events
+- **Pinecone Vector Database**: Store and retrieve vector embeddings for RAG
+- **RAG Question Answering**: Answer questions using relevant information from the vector database
+- **WebSocket Notifications**: Real-time notifications for Admin Bot
+- **API Documentation**: Automatic OpenAPI documentation via Swagger
+- **Docker Support**: Easy deployment using Docker
+- **Auto-Debugging**: Built-in debugging, error tracking, and performance monitoring
+
+## API Endpoints
+
+### MongoDB Endpoints
+
+- `POST /mongodb/session`: Create a new session record
+- `PUT /mongodb/session/{session_id}/response`: Update a session with a response
+- `GET /mongodb/history`: Get user conversation history
+- `GET /mongodb/health`: Check MongoDB connection health
+
+### PostgreSQL Endpoints
+
+- `GET /postgres/health`: Check PostgreSQL connection health
+- `GET /postgres/faq`: Get FAQ items
+- `POST /postgres/faq`: Create a new FAQ item
+- `GET /postgres/faq/{faq_id}`: Get a specific FAQ item
+- `PUT /postgres/faq/{faq_id}`: Update a specific FAQ item
+- `DELETE /postgres/faq/{faq_id}`: Delete a specific FAQ item
+- `GET /postgres/emergency`: Get emergency contact items
+- `POST /postgres/emergency`: Create a new emergency contact item
+- `GET /postgres/emergency/{emergency_id}`: Get a specific emergency contact
+- `GET /postgres/events`: Get event items
+
+### RAG Endpoints
+
+- `POST /rag/chat`: Get answer for a question using RAG
+- `POST /rag/embedding`: Generate embedding for text
+- `GET /rag/health`: Check RAG services health
+
+### WebSocket Endpoints
+
+- `WebSocket /notify`: Receive real-time notifications for new sessions
+
+### Debug Endpoints (Available in Debug Mode Only)
+
+- `GET /debug/config`: Get configuration information
+- `GET /debug/system`: Get system information (CPU, memory, disk usage)
+- `GET /debug/database`: Check all database connections
+- `GET /debug/errors`: View recent error logs
+- `GET /debug/performance`: Get performance metrics
+- `GET /debug/full`: Get comprehensive debug information
+
+## WebSocket API
+
+### Notifications for New Sessions
+
+The backend provides a WebSocket endpoint for receiving notifications about new sessions that match specific criteria.
+
+#### WebSocket Endpoint Configuration
+
+The WebSocket endpoint is configured using environment variables:
+
+```
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+```
+
+The full WebSocket URL will be:
+```
+ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}
+```
+
+For example: `ws://localhost:7860/notify`
+
+#### Notification Criteria
+
+A notification is sent when:
+1. A new session is created with `factor` set to "RAG"
+2. The message content starts with "I don't know"
+
+#### Notification Format
+
+```json
+{
+  "type": "new_session",
+  "timestamp": "2025-04-15 22:30:45",
+  "data": {
+    "session_id": "123e4567-e89b-12d3-a456-426614174000",
+    "factor": "rag",
+    "action": "asking_freely",
+    "created_at": "2025-04-15 22:30:45",
+    "first_name": "John",
+    "last_name": "Doe",
+    "message": "I don't know how to find emergency contacts",
+    "user_id": "12345678",
+    "username": "johndoe"
+  }
+}
+```
+
+#### Usage Example
+
+Admin Bot should establish a WebSocket connection to this endpoint using the configured URL:
+
+```python
+import websocket
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Get WebSocket configuration from environment variables
+WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+
+# Create full URL
+ws_url = f"ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+
+def on_message(ws, message):
+    data = json.loads(message)
+    print(f"Received notification: {data}")
+    # Forward to Telegram Admin
+
+def on_error(ws, error):
+    print(f"Error: {error}")
+
+def on_close(ws, close_status_code, close_msg):
+    print("Connection closed")
+
+def on_open(ws):
+    print("Connection opened")
+    # Send keepalive message periodically
+    ws.send("keepalive")
+
+# Connect to WebSocket
+ws = websocket.WebSocketApp(
+    ws_url,
+    on_open=on_open,
+    on_message=on_message,
+    on_error=on_error,
+    on_close=on_close
+)
+ws.run_forever()
+```
+
+When a notification is received, Admin Bot should forward the content to the Telegram Admin.
+
+## Environment Variables
+
+Create a `.env` file with the following variables:
+
+```
+# PostgreSQL Configuration
+DB_CONNECTION_MODE=aiven
+AIVEN_DB_URL=postgresql://username:password@host:port/dbname?sslmode=require
+
+# MongoDB Configuration
+MONGODB_URL=mongodb+srv://username:password@cluster.mongodb.net/?retryWrites=true&w=majority
+DB_NAME=Telegram
+COLLECTION_NAME=session_chat
+
+# Pinecone configuration
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_INDEX_NAME=your-pinecone-index-name
+PINECONE_ENVIRONMENT=gcp-starter
+
+# Google Gemini API key
+GOOGLE_API_KEY=your-google-api-key
+
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+
+# Application settings
+ENVIRONMENT=production
+DEBUG=false
+PORT=7860
+```
+
+## Installation and Setup
+
+### Local Development
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/ManTT-Data/PixAgent.git
+   cd PixAgent
+   ```
+
+2. Create a virtual environment and install dependencies:
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   pip install -r requirements.txt
+   ```
+
+3. Create a `.env` file with your configuration (see above)
+
+4. Run the application:
+   ```bash
+   uvicorn app:app --reload --port 7860
+   ```
+
+5. Open your browser and navigate to [http://localhost:7860/docs](http://localhost:7860/docs) to see the API documentation
+
+### Docker Deployment
+
+1. Build the Docker image:
+   ```bash
+   docker build -t pix-project-backend .
+   ```
+
+2. Run the Docker container:
+   ```bash
+   docker run -p 7860:7860 --env-file .env pix-project-backend
+   ```
+
+## Deployment to HuggingFace Spaces
+
+1. Create a new Space on HuggingFace (Dockerfile type)
+2. Link your GitHub repository or push directly to the HuggingFace repo
+3. Add your environment variables in the Space settings
+4. The deployment will use `app.py` as the entry point, which is the standard for HuggingFace Spaces
+
+### Important Notes for HuggingFace Deployment
+
+- The application uses `app.py` with the FastAPI instance named `app` to avoid the "Error loading ASGI app. Attribute 'app' not found in module 'app'" error
+- Make sure all environment variables are set in the Space settings
+- The Dockerfile is configured to expose port 7860, which is the default port for HuggingFace Spaces
+
+## Project Structure
+
+```
+.
+├── app                  # Main application package
+│   ├── api              # API endpoints
+│   │   ├── mongodb_routes.py
+│   │   ├── postgresql_routes.py
+│   │   ├── rag_routes.py
+│   │   └── websocket_routes.py
+│   ├── database         # Database connections
+│   │   ├── mongodb.py
+│   │   ├── pinecone.py
+│   │   └── postgresql.py
+│   ├── models           # Pydantic models
+│   │   ├── mongodb_models.py
+│   │   ├── postgresql_models.py
+│   │   └── rag_models.py
+│   └── utils            # Utility functions
+│       ├── debug_utils.py
+│       └── middleware.py
+├── tests                # Test directory
+│   └── test_api_endpoints.py
+├── .dockerignore        # Docker ignore file
+├── .env.example         # Example environment file
+├── .gitattributes       # Git attributes
+├── .gitignore           # Git ignore file
+├── app.py               # Application entry point
+├── docker-compose.yml   # Docker compose configuration
+├── Dockerfile           # Docker configuration
+├── pytest.ini           # Pytest configuration
+├── README.md            # Project documentation
+├── requirements.txt     # Project dependencies
+└── api_documentation.txt # API documentation for frontend engineers
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+# Advanced Retrieval System
+
+This project now features an enhanced vector retrieval system that improves the quality and relevance of information retrieved from Pinecone using threshold-based filtering and multiple similarity metrics.
+
+## Features
+
+### 1. Threshold-Based Retrieval
+
+The system implements a threshold-based approach to vector retrieval, which:
+- Retrieves a larger candidate set from the vector database
+- Applies a similarity threshold to filter out less relevant results
+- Returns only the most relevant documents that exceed the threshold
+
+### 2. Multiple Similarity Metrics
+
+The system supports multiple similarity metrics:
+- **Cosine Similarity** (default): Measures the cosine of the angle between vectors
+- **Dot Product**: Calculates the dot product between vectors
+- **Euclidean Distance**: Measures the straight-line distance between vectors
+
+Each metric has different characteristics and may perform better for different types of data and queries.
+
+### 3. Score Normalization
+
+For metrics like Euclidean distance where lower values indicate higher similarity, the system automatically normalizes scores to a 0-1 scale where higher values always indicate higher similarity. This makes it easier to compare results across different metrics.
+
+## Configuration
+
+The retrieval system can be configured through environment variables:
+
+```
+# Pinecone retrieval configuration
+PINECONE_DEFAULT_LIMIT_K=10       # Maximum number of candidates to retrieve
+PINECONE_DEFAULT_TOP_K=6          # Number of results to return after filtering
+PINECONE_DEFAULT_SIMILARITY_METRIC=cosine  # Default similarity metric
+PINECONE_DEFAULT_SIMILARITY_THRESHOLD=0.75 # Similarity threshold (0-1)
+PINECONE_ALLOWED_METRICS=cosine,dotproduct,euclidean  # Available metrics
+```
+
+## API Usage
+
+You can customize the retrieval parameters when making API requests:
+
+```json
+{
+  "user_id": "user123",
+  "question": "What are the best restaurants in Da Nang?",
+  "similarity_top_k": 5,
+  "limit_k": 15,
+  "similarity_metric": "cosine",
+  "similarity_threshold": 0.8
+}
+```
+
+## Benefits
+
+1. **Quality Improvement**: Retrieves only the most relevant documents above a certain quality threshold
+2. **Flexibility**: Different similarity metrics can be used for different types of queries
+3. **Efficiency**: Avoids processing irrelevant documents, improving response time
+4. **Configurability**: All parameters can be adjusted via environment variables or at request time
+
+## Implementation Details
+
+The system is implemented as a custom retriever class `ThresholdRetriever` that integrates with LangChain's retrieval infrastructure while providing enhanced functionality.
+
+## In-Memory Cache
+
+Dự án bao gồm một hệ thống cache trong bộ nhớ để giảm thiểu truy cập đến cơ sở dữ liệu PostgreSQL và MongoDB.
+
+### Cấu hình Cache
+
+Cache được cấu hình thông qua các biến môi trường:
+
+```
+# Cache Configuration
+CACHE_TTL_SECONDS=300           # Thời gian tồn tại của cache item (giây)
+CACHE_CLEANUP_INTERVAL=60       # Chu kỳ xóa cache hết hạn (giây)
+CACHE_MAX_SIZE=1000             # Số lượng item tối đa trong cache
+HISTORY_QUEUE_SIZE=10           # Số lượng item tối đa trong queue lịch sử người dùng
+HISTORY_CACHE_TTL=3600          # Thời gian tồn tại của lịch sử người dùng (giây)
+```
+
+### Cơ chế Cache
+
+Hệ thống cache kết hợp hai cơ chế hết hạn:
+
+1. **Lazy Expiration**: Kiểm tra thời hạn khi truy cập cache item. Nếu item đã hết hạn, nó sẽ bị xóa và trả về kết quả là không tìm thấy.
+
+2. **Active Expiration**: Một background thread định kỳ quét và xóa các item đã hết hạn. Điều này giúp tránh tình trạng cache quá lớn với các item không còn được sử dụng.
+
+### Các loại dữ liệu được cache
+
+- **Dữ liệu PostgreSQL**: Thông tin từ các bảng FAQ, Emergency Contacts, và Events.
+- **Lịch sử người dùng từ MongoDB**: Lịch sử hội thoại người dùng được lưu trong queue với thời gian sống tính theo lần truy cập cuối cùng.
+
+### API Cache
+
+Dự án cung cấp các API endpoints để quản lý cache:
+
+- `GET /cache/stats`: Xem thống kê về cache (tổng số item, bộ nhớ sử dụng, v.v.)
+- `DELETE /cache/clear`: Xóa toàn bộ cache
+- `GET /debug/cache`: (Chỉ trong chế độ debug) Xem thông tin chi tiết về cache, bao gồm các keys và cấu hình
+
+### Cách hoạt động
+
+1. Khi một request đến, hệ thống sẽ kiểm tra dữ liệu trong cache trước.
+2. Nếu dữ liệu tồn tại và còn hạn, trả về từ cache.
+3. Nếu dữ liệu không tồn tại hoặc đã hết hạn, truy vấn từ database và lưu kết quả vào cache.
+4. Khi dữ liệu được cập nhật hoặc xóa, cache liên quan sẽ tự động được xóa.
+
+### Lịch sử người dùng
+
+Lịch sử hội thoại người dùng được lưu trong queue riêng với cơ chế đặc biệt:
+
+- Mỗi người dùng có một queue riêng với kích thước giới hạn (`HISTORY_QUEUE_SIZE`).
+- Thời gian sống của queue được làm mới mỗi khi có tương tác mới.
+- Khi queue đầy, các item cũ nhất sẽ bị loại bỏ.
+- Queue tự động bị xóa sau một thời gian không hoạt động.
+
+## Tác giả
+
+- **PIX Project Team** 

app.py

@@ -0,0 +1,223 @@
+from fastapi import FastAPI, Depends, Request, HTTPException, status
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+import uvicorn
+import os
+import sys
+import logging
+from dotenv import load_dotenv
+
+# Cấu hình logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+DEBUG = os.getenv("DEBUG", "False").lower() in ("true", "1", "t")
+
+# Kiểm tra các biến môi trường bắt buộc
+required_env_vars = [
+    "AIVEN_DB_URL", 
+    "MONGODB_URL", 
+    "PINECONE_API_KEY", 
+    "PINECONE_INDEX_NAME", 
+    "GOOGLE_API_KEY"
+]
+
+missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+if missing_vars:
+    logger.error(f"Missing required environment variables: {', '.join(missing_vars)}")
+    if not DEBUG:  # Chỉ thoát nếu không ở chế độ debug
+        sys.exit(1)
+
+# Database health checks
+def check_database_connections():
+    """Kiểm tra kết nối các database khi khởi động"""
+    from app.database.postgresql import check_db_connection as check_postgresql
+    from app.database.mongodb import check_db_connection as check_mongodb
+    from app.database.pinecone import check_db_connection as check_pinecone
+    
+    db_status = {
+        "postgresql": check_postgresql(),
+        "mongodb": check_mongodb(),
+        "pinecone": check_pinecone()
+    }
+    
+    all_ok = all(db_status.values())
+    if not all_ok:
+        failed_dbs = [name for name, status in db_status.items() if not status]
+        logger.error(f"Failed to connect to databases: {', '.join(failed_dbs)}")
+        if not DEBUG:  # Chỉ thoát nếu không ở chế độ debug
+            sys.exit(1)
+    
+    return db_status
+
+# Khởi tạo lifespan để kiểm tra kết nối database khi khởi động
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup: kiểm tra kết nối các database
+    logger.info("Starting application...")
+    db_status = check_database_connections()
+    
+    # Khởi tạo bảng trong cơ sở dữ liệu (nếu chưa tồn tại)
+    if DEBUG and all(db_status.values()):  # Chỉ khởi tạo bảng trong chế độ debug và khi tất cả kết nối DB thành công
+        from app.database.postgresql import create_tables
+        if create_tables():
+            logger.info("Database tables created or already exist")
+    
+    yield
+    
+    # Shutdown
+    logger.info("Shutting down application...")
+
+# Import routers
+try:
+    from app.api.mongodb_routes import router as mongodb_router
+    from app.api.postgresql_routes import router as postgresql_router
+    from app.api.rag_routes import router as rag_router
+    from app.api.pdf_routes import router as pdf_router
+    from app.api.websocket_routes import router as websocket_router
+    
+    # Import middlewares
+    from app.utils.middleware import RequestLoggingMiddleware, ErrorHandlingMiddleware, DatabaseCheckMiddleware
+    
+    # Import debug utilities
+    from app.utils.debug_utils import debug_view, DebugInfo, error_tracker, performance_monitor
+    
+    # Import cache
+    from app.utils.cache import get_cache
+    
+except ImportError as e:
+    logger.error(f"Error importing routes or middlewares: {e}")
+    raise
+
+# Create FastAPI app
+app = FastAPI(
+    title="PIX Project Backend API",
+    description="Backend API for PIX Project with MongoDB, PostgreSQL and RAG integration",
+    version="1.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    debug=DEBUG,
+    lifespan=lifespan,
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Thêm middlewares
+app.add_middleware(ErrorHandlingMiddleware)
+app.add_middleware(RequestLoggingMiddleware)
+if not DEBUG:  # Chỉ thêm middleware kiểm tra database trong production
+    app.add_middleware(DatabaseCheckMiddleware)
+
+# Include routers
+app.include_router(mongodb_router)
+app.include_router(postgresql_router)
+app.include_router(rag_router)
+app.include_router(pdf_router)
+app.include_router(websocket_router)
+
+# Root endpoint
+@app.get("/")
+def read_root():
+    return {
+        "message": "Welcome to PIX Project Backend API",
+        "documentation": "/docs",
+    }
+
+# Health check endpoint
+@app.get("/health")
+def health_check():
+    # Kiểm tra kết nối database
+    db_status = check_database_connections()
+    all_db_ok = all(db_status.values())
+    
+    return {
+        "status": "healthy" if all_db_ok else "degraded",
+        "version": "1.0.0",
+        "environment": os.environ.get("ENVIRONMENT", "production"),
+        "databases": db_status
+    }
+
+@app.get("/api/ping")
+async def ping():
+    return {"status": "pong"}
+
+# Cache stats endpoint
+@app.get("/cache/stats")
+def cache_stats():
+    """Trả về thống kê về cache"""
+    cache = get_cache()
+    return cache.stats()
+
+# Cache clear endpoint
+@app.delete("/cache/clear")
+def cache_clear():
+    """Xóa tất cả dữ liệu trong cache"""
+    cache = get_cache()
+    cache.clear()
+    return {"message": "Cache cleared successfully"}
+
+# Debug endpoints (chỉ có trong chế độ debug)
+if DEBUG:
+    @app.get("/debug/config")
+    def debug_config():
+        """Hiển thị thông tin cấu hình (chỉ trong chế độ debug)"""
+        config = {
+            "environment": os.environ.get("ENVIRONMENT", "production"),
+            "debug": DEBUG,
+            "db_connection_mode": os.environ.get("DB_CONNECTION_MODE", "aiven"),
+            "databases": {
+                "postgresql": os.environ.get("AIVEN_DB_URL", "").split("@")[1].split("/")[0] if "@" in os.environ.get("AIVEN_DB_URL", "") else "N/A",
+                "mongodb": os.environ.get("MONGODB_URL", "").split("@")[1].split("/?")[0] if "@" in os.environ.get("MONGODB_URL", "") else "N/A",
+                "pinecone": os.environ.get("PINECONE_INDEX_NAME", "N/A"),
+            }
+        }
+        return config
+    
+    @app.get("/debug/system")
+    def debug_system():
+        """Hiển thị thông tin hệ thống (chỉ trong chế độ debug)"""
+        return DebugInfo.get_system_info()
+    
+    @app.get("/debug/database")
+    def debug_database():
+        """Hiển thị trạng thái database (chỉ trong chế độ debug)"""
+        return DebugInfo.get_database_status()
+    
+    @app.get("/debug/errors")
+    def debug_errors(limit: int = 10):
+        """Hiển thị các lỗi gần đây (chỉ trong chế độ debug)"""
+        return error_tracker.get_recent_errors(limit)
+    
+    @app.get("/debug/performance")
+    def debug_performance():
+        """Hiển thị thống kê hiệu suất (chỉ trong chế độ debug)"""
+        return performance_monitor.get_stats()
+    
+    @app.get("/debug/full")
+    def debug_full_report(request: Request):
+        """Hiển thị báo cáo đầy đủ về hệ thống (chỉ trong chế độ debug)"""
+        return debug_view(request)
+    
+    @app.get("/debug/cache")
+    def debug_cache():
+        """Hiển thị thống kê về cache (chỉ trong chế độ debug)"""
+        return get_cache().stats()
+
+if __name__ == "__main__":
+    PORT = int(os.getenv("PORT", "7860"))
+    uvicorn.run("app:app", host="0.0.0.0", port=PORT, reload=DEBUG) 

app/__init__.py

@@ -0,0 +1,25 @@
+# PIX Project Backend
+# Version: 1.0.0
+
+__version__ = "1.0.0"
+
+# Import app từ app.py để tests có thể tìm thấy
+import sys
+import os
+
+# Thêm thư mục gốc vào sys.path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+try:
+    # Sửa lại cách import đúng - 'app.py' không phải là module hợp lệ
+    # 'app' là tên module, '.py' là phần mở rộng tệp
+    from app import app
+except ImportError:
+    # Thử cách khác nếu import trực tiếp không hoạt động
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("app_module", 
+                                                 os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 
+                                                             "app.py"))
+    app_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(app_module)
+    app = app_module.app 

app/api/__init__.py

@@ -0,0 +1 @@
+# API routes package 

app/api/mongodb_routes.py

@@ -0,0 +1,276 @@
+from fastapi import APIRouter, HTTPException, Depends, Query, status, Response
+from typing import List, Optional, Dict
+from pymongo.errors import PyMongoError
+import logging
+from datetime import datetime
+import traceback
+import asyncio
+
+from app.database.mongodb import (
+    save_session, 
+    get_chat_history,
+    update_session_response,
+    check_db_connection,
+    session_collection
+)
+from app.models.mongodb_models import (
+    SessionCreate,
+    SessionResponse,
+    HistoryRequest,
+    HistoryResponse,
+    QuestionAnswer
+)
+from app.api.websocket_routes import send_notification
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Create router
+router = APIRouter(
+    prefix="/mongodb",
+    tags=["MongoDB"],
+)
+
+@router.post("/session", response_model=SessionResponse, status_code=status.HTTP_201_CREATED)
+async def create_session(session: SessionCreate, response: Response):
+    """
+    Create a new session record in MongoDB.
+    
+    - **session_id**: Unique identifier for the session (auto-generated if not provided)
+    - **factor**: Factor type (user, rag, etc.)
+    - **action**: Action type (start, events, faq, emergency, help, asking_freely, etc.)
+    - **first_name**: User's first name
+    - **last_name**: User's last name (optional)
+    - **message**: User's message (optional)
+    - **user_id**: User's ID from Telegram
+    - **username**: User's username (optional)
+    - **response**: Response from RAG (optional)
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to create session")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Create new session in MongoDB
+        result = save_session(
+            session_id=session.session_id,
+            factor=session.factor,
+            action=session.action,
+            first_name=session.first_name,
+            last_name=session.last_name,
+            message=session.message,
+            user_id=session.user_id,
+            username=session.username,
+            response=session.response
+        )
+        
+        # Chuẩn bị response object
+        session_response = SessionResponse(
+            **session.model_dump(),
+            created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        )
+        
+        # Kiểm tra nếu session cần gửi thông báo (response bắt đầu bằng "I'm sorry")
+        if session.response and session.response.strip().lower().startswith("i'm sorry"):
+            # Gửi thông báo qua WebSocket
+            try:
+                notification_data = {
+                    "session_id": session.session_id,
+                    "factor": session.factor,
+                    "action": session.action,
+                    "message": session.message,
+                    "user_id": session.user_id,
+                    "username": session.username,
+                    "first_name": session.first_name,
+                    "last_name": session.last_name,
+                    "response": session.response,
+                    "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                }
+                
+                # Khởi tạo task để gửi thông báo - sử dụng asyncio.create_task để đảm bảo không block quá trình chính
+                asyncio.create_task(send_notification(notification_data))
+                logger.info(f"Notification queued for session {session.session_id} - response starts with 'I'm sorry'")
+            except Exception as e:
+                logger.error(f"Error queueing notification: {e}")
+                # Không dừng xử lý chính khi gửi thông báo thất bại
+        
+        # Return response
+        return session_response
+    except PyMongoError as e:
+        logger.error(f"MongoDB error creating session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error creating session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create session: {str(e)}"
+        )
+
+@router.put("/session/{session_id}/response", status_code=status.HTTP_200_OK)
+async def update_session_with_response(session_id: str, response_text: str):
+    """
+    Update a session with the response.
+    
+    - **session_id**: ID of the session to update
+    - **response_text**: Response to add to the session
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to update session response")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Update session in MongoDB
+        result = update_session_response(session_id, response_text)
+        
+        if not result:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Session with ID {session_id} not found"
+            )
+            
+        return {"status": "success", "message": "Response added to session"}
+    except PyMongoError as e:
+        logger.error(f"MongoDB error updating session response: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except HTTPException:
+        # Re-throw HTTP exceptions
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error updating session response: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to update session: {str(e)}"
+        )
+
+@router.get("/history", response_model=HistoryResponse)
+async def get_history(user_id: str, n: int = Query(3, ge=1, le=10)):
+    """
+    Get user history for a specific user.
+    
+    - **user_id**: User's ID from Telegram
+    - **n**: Number of most recent interactions to return (default: 3, min: 1, max: 10)
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to get user history")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Get user history from MongoDB
+        history_data = get_chat_history(user_id=user_id, n=n)
+        
+        # Convert to response model
+        return HistoryResponse(history=history_data)
+    except PyMongoError as e:
+        logger.error(f"MongoDB error getting user history: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error getting user history: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to get user history: {str(e)}"
+        )
+
+@router.get("/health")
+async def health_check():
+    """
+    Check health of MongoDB connection.
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        is_connected = check_db_connection()
+        
+        if not is_connected:
+            return {
+                "status": "unhealthy", 
+                "message": "MongoDB connection failed", 
+                "timestamp": datetime.now().isoformat()
+            }
+            
+        return {
+            "status": "healthy", 
+            "message": "MongoDB connection is working", 
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        logger.error(f"MongoDB health check failed: {e}")
+        logger.error(traceback.format_exc())
+        return {
+            "status": "error", 
+            "message": f"MongoDB health check error: {str(e)}", 
+            "timestamp": datetime.now().isoformat()
+        }
+
+@router.get("/session/{session_id}")
+async def get_session(session_id: str):
+    """
+    Lấy thông tin session từ MongoDB theo session_id.
+    
+    - **session_id**: ID của session cần lấy
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to get session")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Lấy thông tin từ MongoDB
+        session_data = session_collection.find_one({"session_id": session_id})
+        
+        if not session_data:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Session with ID {session_id} not found"
+            )
+            
+        # Chuyển _id thành string để có thể JSON serialize
+        if "_id" in session_data:
+            session_data["_id"] = str(session_data["_id"])
+            
+        return session_data
+    except PyMongoError as e:
+        logger.error(f"MongoDB error getting session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except HTTPException:
+        # Re-throw HTTP exceptions
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error getting session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to get session: {str(e)}"
+        ) 

app/api/pdf_routes.py

@@ -0,0 +1,310 @@
+import os
+import shutil
+import uuid
+from fastapi import APIRouter, UploadFile, File, Form, HTTPException, BackgroundTasks, Depends
+from fastapi.responses import JSONResponse
+from typing import Optional, List, Dict, Any
+from sqlalchemy.orm import Session
+
+from app.utils.pdf_processor import PDFProcessor
+from app.models.pdf_models import PDFResponse, DeleteDocumentRequest, DocumentsListResponse
+from app.database.postgresql import get_db
+from app.database.models import VectorDatabase, Document, VectorStatus, DocumentContent
+from datetime import datetime
+from app.api.pdf_websocket import (
+    send_pdf_upload_started, 
+    send_pdf_upload_progress, 
+    send_pdf_upload_completed,
+    send_pdf_upload_failed,
+    send_pdf_delete_started,
+    send_pdf_delete_completed,
+    send_pdf_delete_failed
+)
+
+# Khởi tạo router
+router = APIRouter(
+    prefix="/pdf",
+    tags=["PDF Processing"],
+)
+
+# Thư mục lưu file tạm - sử dụng /tmp để tránh lỗi quyền truy cập
+TEMP_UPLOAD_DIR = "/tmp/uploads/temp"
+STORAGE_DIR = "/tmp/uploads/pdfs"
+
+# Đảm bảo thư mục upload tồn tại
+os.makedirs(TEMP_UPLOAD_DIR, exist_ok=True)
+os.makedirs(STORAGE_DIR, exist_ok=True)
+
+# Endpoint upload và xử lý PDF
+@router.post("/upload", response_model=PDFResponse)
+async def upload_pdf(
+    file: UploadFile = File(...),
+    namespace: str = Form("Default"),
+    index_name: str = Form("testbot768"),
+    title: Optional[str] = Form(None),
+    description: Optional[str] = Form(None),
+    user_id: Optional[str] = Form(None),
+    vector_database_id: Optional[int] = Form(None),
+    background_tasks: BackgroundTasks = None,
+    db: Session = Depends(get_db)
+):
+    """
+    Upload và xử lý file PDF để tạo embeddings và lưu vào Pinecone
+    
+    - **file**: File PDF cần xử lý
+    - **namespace**: Namespace trong Pinecone để lưu embeddings (mặc định: "Default")
+    - **index_name**: Tên index Pinecone (mặc định: "testbot768")
+    - **title**: Tiêu đề của tài liệu (tùy chọn)
+    - **description**: Mô tả về tài liệu (tùy chọn)
+    - **user_id**: ID của người dùng để cập nhật trạng thái qua WebSocket
+    - **vector_database_id**: ID của vector database trong PostgreSQL (tùy chọn)
+    """
+    try:
+        # Kiểm tra file có phải PDF không
+        if not file.filename.lower().endswith('.pdf'):
+            raise HTTPException(status_code=400, detail="Chỉ chấp nhận file PDF")
+        
+        # Nếu có vector_database_id, lấy thông tin từ PostgreSQL
+        api_key = None
+        vector_db = None
+        
+        if vector_database_id:
+            vector_db = db.query(VectorDatabase).filter(
+                VectorDatabase.id == vector_database_id,
+                VectorDatabase.status == "active"
+            ).first()
+            
+            if not vector_db:
+                raise HTTPException(status_code=404, detail="Vector database không tồn tại hoặc không hoạt động")
+            
+            # Sử dụng thông tin từ vector database
+            api_key = vector_db.api_key
+            index_name = vector_db.pinecone_index
+        
+        # Tạo file_id và lưu file tạm
+        file_id = str(uuid.uuid4())
+        temp_file_path = os.path.join(TEMP_UPLOAD_DIR, f"{file_id}.pdf")
+        
+        # Gửi thông báo bắt đầu xử lý qua WebSocket nếu có user_id
+        if user_id:
+            await send_pdf_upload_started(user_id, file.filename, file_id)
+        
+        # Lưu file
+        file_content = await file.read()
+        with open(temp_file_path, "wb") as buffer:
+            buffer.write(file_content)
+            
+        # Tạo metadata
+        metadata = {
+            "filename": file.filename,
+            "content_type": file.content_type
+        }
+        
+        if title:
+            metadata["title"] = title
+        if description:
+            metadata["description"] = description
+        
+        # Gửi thông báo tiến độ qua WebSocket
+        if user_id:
+            await send_pdf_upload_progress(
+                user_id, 
+                file_id, 
+                "file_preparation", 
+                0.2, 
+                "File saved, preparing for processing"
+            )
+        
+        # Lưu thông tin tài liệu vào PostgreSQL nếu có vector_database_id
+        if vector_database_id and vector_db:
+            # Create document record without file content
+            document = Document(
+                name=title or file.filename,
+                file_type="pdf",
+                content_type=file.content_type,
+                size=len(file_content),
+                is_embedded=False,
+                vector_database_id=vector_database_id
+            )
+            db.add(document)
+            db.commit()
+            db.refresh(document)
+            
+            # Create document content record to store binary data separately
+            document_content = DocumentContent(
+                document_id=document.id,
+                file_content=file_content
+            )
+            db.add(document_content)
+            db.commit()
+            
+            # Tạo vector status record
+            vector_status = VectorStatus(
+                document_id=document.id,
+                vector_database_id=vector_database_id,
+                status="pending"
+            )
+            db.add(vector_status)
+            db.commit()
+            
+        # Khởi tạo PDF processor với API key nếu có
+        processor = PDFProcessor(index_name=index_name, namespace=namespace, api_key=api_key)
+        
+        # Gửi thông báo bắt đầu embedding qua WebSocket
+        if user_id:
+            await send_pdf_upload_progress(
+                user_id, 
+                file_id, 
+                "embedding_start", 
+                0.4, 
+                "Starting to process PDF and create embeddings"
+            )
+        
+        # Xử lý PDF và tạo embeddings
+        # Tạo callback function để xử lý cập nhật tiến độ
+        async def progress_callback_wrapper(step, progress, message):
+            if user_id:
+                await send_progress_update(user_id, file_id, step, progress, message)
+        
+        # Xử lý PDF và tạo embeddings với callback đã được xử lý đúng cách
+        result = await processor.process_pdf(
+            file_path=temp_file_path,
+            document_id=file_id,
+            metadata=metadata,
+            progress_callback=progress_callback_wrapper
+        )
+        
+        # Nếu thành công, chuyển file vào storage
+        if result.get('success'):
+            storage_path = os.path.join(STORAGE_DIR, f"{file_id}.pdf")
+            shutil.move(temp_file_path, storage_path)
+            
+            # Cập nhật trạng thái trong PostgreSQL nếu có vector_database_id
+            if vector_database_id and 'document' in locals() and 'vector_status' in locals():
+                vector_status.status = "completed"
+                vector_status.embedded_at = datetime.now()
+                vector_status.vector_id = file_id
+                document.is_embedded = True
+                db.commit()
+            
+            # Gửi thông báo hoàn thành qua WebSocket
+            if user_id:
+                await send_pdf_upload_completed(
+                    user_id,
+                    file_id,
+                    file.filename,
+                    result.get('chunks_processed', 0)
+                )
+        else:
+            # Cập nhật trạng thái lỗi trong PostgreSQL nếu có vector_database_id
+            if vector_database_id and 'vector_status' in locals():
+                vector_status.status = "failed"
+                vector_status.error_message = result.get('error', 'Unknown error')
+                db.commit()
+                
+            # Gửi thông báo lỗi qua WebSocket
+            if user_id:
+                await send_pdf_upload_failed(
+                    user_id,
+                    file_id,
+                    file.filename,
+                    result.get('error', 'Unknown error')
+                )
+            
+        # Dọn dẹp: xóa file tạm nếu vẫn còn
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+            
+        return result
+    except Exception as e:
+        # Dọn dẹp nếu có lỗi
+        if 'temp_file_path' in locals() and os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+            
+        # Cập nhật trạng thái lỗi trong PostgreSQL nếu có vector_database_id
+        if 'vector_database_id' in locals() and vector_database_id and 'vector_status' in locals():
+            vector_status.status = "failed"
+            vector_status.error_message = str(e)
+            db.commit()
+            
+        # Gửi thông báo lỗi qua WebSocket
+        if 'user_id' in locals() and user_id and 'file_id' in locals():
+            await send_pdf_upload_failed(
+                user_id,
+                file_id,
+                file.filename,
+                str(e)
+            )
+            
+        return PDFResponse(
+            success=False,
+            error=str(e)
+        )
+
+# Function để gửi cập nhật tiến độ - được sử dụng trong callback
+async def send_progress_update(user_id, document_id, step, progress, message):
+    if user_id:
+        await send_pdf_upload_progress(user_id, document_id, step, progress, message)
+
+# Endpoint xóa tài liệu
+@router.delete("/namespace", response_model=PDFResponse)
+async def delete_namespace(
+    namespace: str = "Default",
+    index_name: str = "testbot768",
+    user_id: Optional[str] = None
+):
+    """
+    Xóa toàn bộ embeddings trong một namespace từ Pinecone (tương ứng xoá namespace)
+
+    - **namespace**: Namespace trong Pinecone (mặc định: "Default")
+    - **index_name**: Tên index Pinecone (mặc định: "testbot768")
+    - **user_id**: ID của người dùng để cập nhật trạng thái qua WebSocket
+    """
+    try:
+        # Gửi thông báo bắt đầu xóa qua WebSocket
+        if user_id:
+            await send_pdf_delete_started(user_id, namespace)
+            
+        processor = PDFProcessor(index_name=index_name, namespace=namespace)
+        result = await processor.delete_namespace()
+        
+        # Gửi thông báo kết quả qua WebSocket
+        if user_id:
+            if result.get('success'):
+                await send_pdf_delete_completed(user_id, namespace)
+            else:
+                await send_pdf_delete_failed(user_id, namespace, result.get('error', 'Unknown error'))
+                
+        return result
+    except Exception as e:
+        # Gửi thông báo lỗi qua WebSocket
+        if user_id:
+            await send_pdf_delete_failed(user_id, namespace, str(e))
+            
+        return PDFResponse(
+            success=False,
+            error=str(e)
+        )
+
+# Endpoint lấy danh sách tài liệu
+@router.get("/documents", response_model=DocumentsListResponse)
+async def get_documents(namespace: str = "Default", index_name: str = "testbot768"):
+    """
+    Lấy thông tin về tất cả tài liệu đã được embed
+    
+    - **namespace**: Namespace trong Pinecone (mặc định: "Default")
+    - **index_name**: Tên index Pinecone (mặc định: "testbot768")
+    """
+    try:
+        # Khởi tạo PDF processor
+        processor = PDFProcessor(index_name=index_name, namespace=namespace)
+        
+        # Lấy danh sách documents
+        result = await processor.list_documents()
+        
+        return result
+    except Exception as e:
+        return DocumentsListResponse(
+            success=False,
+            error=str(e)
+        ) 

app/api/pdf_websocket.py

@@ -0,0 +1,263 @@
+import logging
+from typing import Dict, List, Optional, Any
+from fastapi import WebSocket, WebSocketDisconnect, APIRouter
+from pydantic import BaseModel
+import json
+import time
+
+# Cấu hình logging
+logger = logging.getLogger(__name__)
+
+# Models cho Swagger documentation
+class ConnectionStatus(BaseModel):
+    user_id: str
+    active: bool
+    connection_count: int
+    last_activity: Optional[float] = None
+
+class UserConnection(BaseModel):
+    user_id: str
+    connection_count: int
+
+class AllConnectionsStatus(BaseModel):
+    total_users: int
+    total_connections: int
+    users: List[UserConnection]
+
+# Khởi tạo router
+router = APIRouter(
+    prefix="/ws",
+    tags=["WebSockets"],
+)
+
+class ConnectionManager:
+    """Quản lý các kết nối WebSocket"""
+    
+    def __init__(self):
+        # Lưu trữ các kết nối theo user_id
+        self.active_connections: Dict[str, List[WebSocket]] = {}
+        
+    async def connect(self, websocket: WebSocket, user_id: str):
+        """Kết nối một WebSocket mới"""
+        await websocket.accept()
+        if user_id not in self.active_connections:
+            self.active_connections[user_id] = []
+        self.active_connections[user_id].append(websocket)
+        logger.info(f"New WebSocket connection for user {user_id}. Total connections: {len(self.active_connections[user_id])}")
+        
+    def disconnect(self, websocket: WebSocket, user_id: str):
+        """Ngắt kết nối WebSocket"""
+        if user_id in self.active_connections:
+            if websocket in self.active_connections[user_id]:
+                self.active_connections[user_id].remove(websocket)
+            # Xóa user_id khỏi dict nếu không còn kết nối nào
+            if not self.active_connections[user_id]:
+                del self.active_connections[user_id]
+        logger.info(f"WebSocket disconnected for user {user_id}")
+        
+    async def send_message(self, message: Dict[str, Any], user_id: str):
+        """Gửi tin nhắn tới tất cả kết nối của một user"""
+        if user_id in self.active_connections:
+            disconnected_websockets = []
+            for websocket in self.active_connections[user_id]:
+                try:
+                    await websocket.send_text(json.dumps(message))
+                except Exception as e:
+                    logger.error(f"Error sending message to WebSocket: {str(e)}")
+                    disconnected_websockets.append(websocket)
+            
+            # Xóa các kết nối bị ngắt
+            for websocket in disconnected_websockets:
+                self.disconnect(websocket, user_id)
+    
+    def get_connection_status(self, user_id: str = None) -> Dict[str, Any]:
+        """Lấy thông tin về trạng thái kết nối WebSocket"""
+        if user_id:
+            # Trả về thông tin kết nối cho user cụ thể
+            if user_id in self.active_connections:
+                return {
+                    "user_id": user_id,
+                    "active": True,
+                    "connection_count": len(self.active_connections[user_id]),
+                    "last_activity": time.time()
+                }
+            else:
+                return {
+                    "user_id": user_id,
+                    "active": False,
+                    "connection_count": 0,
+                    "last_activity": None
+                }
+        else:
+            # Trả về thông tin tất cả kết nối
+            result = {
+                "total_users": len(self.active_connections),
+                "total_connections": sum(len(connections) for connections in self.active_connections.values()),
+                "users": []
+            }
+            
+            for uid, connections in self.active_connections.items():
+                result["users"].append({
+                    "user_id": uid,
+                    "connection_count": len(connections)
+                })
+            
+            return result
+
+
+# Tạo instance của ConnectionManager
+manager = ConnectionManager()
+
+@router.websocket("/pdf/{user_id}")
+async def websocket_endpoint(websocket: WebSocket, user_id: str):
+    """Endpoint WebSocket để cập nhật tiến trình xử lý PDF"""
+    await manager.connect(websocket, user_id)
+    try:
+        while True:
+            # Đợi tin nhắn từ client (chỉ để giữ kết nối)
+            await websocket.receive_text()
+    except WebSocketDisconnect:
+        manager.disconnect(websocket, user_id)
+    except Exception as e:
+        logger.error(f"WebSocket error: {str(e)}")
+        manager.disconnect(websocket, user_id)
+
+# API endpoints để kiểm tra trạng thái WebSocket
+@router.get("/status", response_model=AllConnectionsStatus, responses={
+    200: {
+        "description": "Successful response",
+        "content": {
+            "application/json": {
+                "example": {
+                    "total_users": 2,
+                    "total_connections": 3,
+                    "users": [
+                        {"user_id": "user1", "connection_count": 2},
+                        {"user_id": "user2", "connection_count": 1}
+                    ]
+                }
+            }
+        }
+    }
+})
+async def get_all_websocket_connections():
+    """
+    Lấy thông tin về tất cả kết nối WebSocket hiện tại.
+    
+    Endpoint này trả về:
+    - Tổng số người dùng đang kết nối
+    - Tổng số kết nối WebSocket
+    - Danh sách người dùng kèm theo số lượng kết nối của mỗi người
+    """
+    return manager.get_connection_status()
+
+@router.get("/status/{user_id}", response_model=ConnectionStatus, responses={
+    200: {
+        "description": "Successful response for active connection",
+        "content": {
+            "application/json": {
+                "examples": {
+                    "active_connection": {
+                        "summary": "Active connection",
+                        "value": {
+                            "user_id": "user123",
+                            "active": True,
+                            "connection_count": 2,
+                            "last_activity": 1634567890.123
+                        }
+                    },
+                    "no_connection": {
+                        "summary": "No active connection",
+                        "value": {
+                            "user_id": "user456",
+                            "active": False,
+                            "connection_count": 0,
+                            "last_activity": None
+                        }
+                    }
+                }
+            }
+        }
+    }
+})
+async def get_user_websocket_status(user_id: str):
+    """
+    Lấy thông tin về kết nối WebSocket của một người dùng cụ thể.
+    
+    Parameters:
+    - **user_id**: ID của người dùng cần kiểm tra
+    
+    Returns:
+    - Thông tin về trạng thái kết nối, bao gồm:
+      - active: Có đang kết nối hay không
+      - connection_count: Số lượng kết nối hiện tại
+      - last_activity: Thời gian hoạt động gần nhất
+    """
+    return manager.get_connection_status(user_id)
+
+# Các hàm gửi thông báo cập nhật trạng thái
+
+async def send_pdf_upload_started(user_id: str, filename: str, document_id: str):
+    """Gửi thông báo bắt đầu upload PDF"""
+    await manager.send_message({
+        "type": "pdf_upload_started",
+        "document_id": document_id,
+        "filename": filename,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_upload_progress(user_id: str, document_id: str, step: str, progress: float, message: str):
+    """Gửi thông báo tiến độ upload PDF"""
+    await manager.send_message({
+        "type": "pdf_upload_progress",
+        "document_id": document_id,
+        "step": step,
+        "progress": progress,
+        "message": message,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_upload_completed(user_id: str, document_id: str, filename: str, chunks: int):
+    """Gửi thông báo hoàn thành upload PDF"""
+    await manager.send_message({
+        "type": "pdf_upload_completed",
+        "document_id": document_id,
+        "filename": filename,
+        "chunks": chunks,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_upload_failed(user_id: str, document_id: str, filename: str, error: str):
+    """Gửi thông báo lỗi upload PDF"""
+    await manager.send_message({
+        "type": "pdf_upload_failed",
+        "document_id": document_id,
+        "filename": filename,
+        "error": error,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_delete_started(user_id: str, namespace: str):
+    """Gửi thông báo bắt đầu xóa PDF"""
+    await manager.send_message({
+        "type": "pdf_delete_started",
+        "namespace": namespace,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_delete_completed(user_id: str, namespace: str):
+    """Gửi thông báo hoàn thành xóa PDF"""
+    await manager.send_message({
+        "type": "pdf_delete_completed",
+        "namespace": namespace,
+        "timestamp": int(time.time())
+    }, user_id)
+
+async def send_pdf_delete_failed(user_id: str, namespace: str, error: str):
+    """Gửi thông báo lỗi xóa PDF"""
+    await manager.send_message({
+        "type": "pdf_delete_failed",
+        "namespace": namespace,
+        "error": error,
+        "timestamp": int(time.time())
+    }, user_id)

app/api/rag_routes.py

@@ -0,0 +1,338 @@
+from fastapi import APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request
+from typing import List, Optional, Dict, Any
+import logging
+import time
+import os
+import json
+import hashlib
+import asyncio
+import traceback
+import google.generativeai as genai
+from datetime import datetime
+from langchain.prompts import PromptTemplate
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from app.utils.utils import timer_decorator
+
+from app.database.mongodb import get_chat_history, get_request_history, session_collection
+from app.database.pinecone import (
+    search_vectors, 
+    get_chain, 
+    DEFAULT_TOP_K,
+    DEFAULT_LIMIT_K,
+    DEFAULT_SIMILARITY_METRIC,
+    DEFAULT_SIMILARITY_THRESHOLD,
+    ALLOWED_METRICS
+)
+from app.models.rag_models import (
+    ChatRequest,
+    ChatResponse,
+    ChatResponseInternal,
+    SourceDocument,
+    EmbeddingRequest,
+    EmbeddingResponse,
+    UserMessageModel
+)
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Configure Google Gemini API
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+genai.configure(api_key=GOOGLE_API_KEY)
+
+# Create router
+router = APIRouter(
+    prefix="/rag",
+    tags=["RAG"],
+)
+
+fix_request = PromptTemplate(
+    template = """Goal:
+Your task is to extract important keywords from the user's current request, optionally using chat history if relevant.
+You will receive a conversation history and the user's current message.
+Generate a **list of concise keywords** that best represent the user's intent.
+
+Return Format:
+Only return keywords (comma-separated, no extra explanation).
+If the current message is NOT related to the chat history or if there is no chat history: Return keywords from the current message only.
+If the current message IS related to the chat history: Return a refined set of keywords based on both history and current message.
+
+Warning:
+Only use chat history if the current message is clearly related to the prior context.
+
+Conversation History:
+{chat_history}
+
+User current message:
+{question}
+""",
+    input_variables=["chat_history", "question"],
+)
+
+# Create a prompt template with conversation history
+prompt = PromptTemplate(
+    template = """Goal:
+You are a professional tour guide assistant that assists users in finding information about places in Da Nang, Vietnam.
+You can provide details on restaurants, cafes, hotels, attractions, and other local venues. 
+You have to use core knowledge and conversation history to chat with users, who are Da Nang's tourists. 
+
+Return Format:
+Respond in friendly, natural, concise and use only English like a real tour guide.
+Always use HTML tags (e.g. <b> for bold) so that Telegram can render the special formatting correctly.
+
+Warning:
+Let's support users like a real tour guide, not a bot. The information in core knowledge is your own knowledge.
+Your knowledge is provided in the Core Knowledge. All of information in Core Knowledge is about Da Nang, Vietnam.
+You just care about current time that user mention when user ask about Solana event.
+Only use core knowledge to answer. If you do not have enough information to answer user's question, please reply with "I'm sorry. I don't have information about that" and Give users some more options to ask.
+
+Core knowledge:
+{context}
+
+Conversation History:
+{chat_history}
+
+User message:
+{question}
+
+Your message:
+""",
+    input_variables = ["context", "question", "chat_history"],
+)
+
+# Helper for embeddings
+async def get_embedding(text: str):
+    """Get embedding from Google Gemini API"""
+    try:
+        # Initialize embedding model
+        embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        
+        # Generate embedding
+        result = await embedding_model.aembed_query(text)
+        
+        # Return embedding
+        return {
+            "embedding": result,
+            "text": text,
+            "model": "embedding-001"
+        }
+    except Exception as e:
+        logger.error(f"Error generating embedding: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to generate embedding: {str(e)}")
+
+# Endpoint for generating embeddings
+@router.post("/embedding", response_model=EmbeddingResponse)
+async def create_embedding(request: EmbeddingRequest):
+    """
+    Generate embedding for text.
+    
+    - **text**: Text to generate embedding for
+    """
+    try:
+        # Get embedding
+        embedding_data = await get_embedding(request.text)
+        
+        # Return embedding
+        return EmbeddingResponse(**embedding_data)
+    except Exception as e:
+        logger.error(f"Error generating embedding: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to generate embedding: {str(e)}")
+
+@timer_decorator
+@router.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest, background_tasks: BackgroundTasks):
+    """
+    Get answer for a question using RAG.
+    
+    - **user_id**: User's ID from Telegram
+    - **question**: User's question
+    - **include_history**: Whether to include user history in prompt (default: True)
+    - **use_rag**: Whether to use RAG (default: True)
+    - **similarity_top_k**: Number of top similar documents to return after filtering (default: 6)
+    - **limit_k**: Maximum number of documents to retrieve from vector store (default: 10)
+    - **similarity_metric**: Similarity metric to use - cosine, dotproduct, euclidean (default: cosine)
+    - **similarity_threshold**: Threshold for vector similarity (default: 0.75)
+    - **session_id**: Optional session ID for tracking conversations
+    - **first_name**: User's first name
+    - **last_name**: User's last name
+    - **username**: User's username
+    """
+    start_time = time.time()
+    try:
+        # Save user message first (so it's available for user history)
+        session_id = request.session_id or f"{request.user_id}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}"
+        # logger.info(f"Processing chat request for user {request.user_id}, session {session_id}")
+
+        retriever = get_chain(
+            top_k=request.similarity_top_k,
+            limit_k=request.limit_k,
+            similarity_metric=request.similarity_metric,
+            similarity_threshold=request.similarity_threshold
+        )
+        if not retriever:
+            raise HTTPException(status_code=500, detail="Failed to initialize retriever")
+        
+        # Get chat history
+        chat_history = get_chat_history(request.user_id) if request.include_history else ""
+        logger.info(f"Using chat history: {chat_history[:100]}...")
+        
+        # Initialize Gemini model
+        generation_config = {
+            "temperature": 0.9,
+            "top_p": 1,
+            "top_k": 1,
+            "max_output_tokens": 2048,
+        }
+
+        safety_settings = [
+            {
+                "category": "HARM_CATEGORY_HARASSMENT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_HATE_SPEECH",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+        ]
+
+        model = genai.GenerativeModel(
+            model_name='models/gemini-2.0-flash',
+            generation_config=generation_config,
+            safety_settings=safety_settings
+        )
+
+        prompt_request = fix_request.format(
+            question=request.question,
+            chat_history=chat_history
+        )
+        
+        # Log thời gian bắt đầu final_request
+        final_request_start_time = time.time()
+        final_request = model.generate_content(prompt_request)
+        # Log thời gian hoàn thành final_request
+        logger.info(f"Fixed Request: {final_request.text}")
+        logger.info(f"Final request generation time: {time.time() - final_request_start_time:.2f} seconds")
+        # print(final_request.text)
+
+        retrieved_docs = retriever.invoke(final_request.text)
+        logger.info(f"Retrieve: {retrieved_docs}")
+        context = "\n".join([doc.page_content for doc in retrieved_docs])
+
+        sources = []
+        for doc in retrieved_docs:
+            source = None
+            metadata = {}
+            
+            if hasattr(doc, 'metadata'):
+                source = doc.metadata.get('source', None)
+                # Extract score information
+                score = doc.metadata.get('score', None)
+                normalized_score = doc.metadata.get('normalized_score', None)
+                # Remove score info from metadata to avoid duplication
+                metadata = {k: v for k, v in doc.metadata.items() 
+                            if k not in ['text', 'source', 'score', 'normalized_score']}
+            
+            sources.append(SourceDocument(
+                text=doc.page_content,
+                source=source,
+                score=score,
+                normalized_score=normalized_score,
+                metadata=metadata
+            ))
+        
+        # Generate the prompt using template
+        prompt_text = prompt.format(
+            context=context,
+            question=final_request.text,
+            chat_history=chat_history
+        )
+        logger.info(f"Full prompt with history and context: {prompt_text}")
+        
+        # Generate response
+        response = model.generate_content(prompt_text)
+        answer = response.text
+        
+        # Calculate processing time
+        processing_time = time.time() - start_time
+        
+        # Log full response with sources
+        # logger.info(f"Generated response for user {request.user_id}: {answer}")
+        
+        # Create response object for API (without sources)
+        chat_response = ChatResponse(
+            answer=answer,
+            processing_time=processing_time
+        )
+        
+        # Return response
+        return chat_response
+    except Exception as e:
+        logger.error(f"Error processing chat request: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Failed to process chat request: {str(e)}")
+
+# Health check endpoint
+@router.get("/health")
+async def health_check():
+    """
+    Check health of RAG services and retrieval system.
+    
+    Returns:
+        - status: "healthy" if all services are working, "degraded" otherwise
+        - services: Status of each service (gemini, pinecone)
+        - retrieval_config: Current retrieval configuration
+        - timestamp: Current time
+    """
+    services = {
+        "gemini": False,
+        "pinecone": False
+    }
+    
+    # Check Gemini
+    try:
+        # Initialize simple model
+        model = genai.GenerativeModel("gemini-2.0-flash")
+        # Test generation
+        response = model.generate_content("Hello")
+        services["gemini"] = True
+    except Exception as e:
+        logger.error(f"Gemini health check failed: {e}")
+    
+    # Check Pinecone
+    try:
+        # Import pinecone function
+        from app.database.pinecone import get_pinecone_index
+        # Get index
+        index = get_pinecone_index()
+        # Check if index exists
+        if index:
+            services["pinecone"] = True
+    except Exception as e:
+        logger.error(f"Pinecone health check failed: {e}")
+    
+    # Get retrieval configuration
+    retrieval_config = {
+        "default_top_k": DEFAULT_TOP_K,
+        "default_limit_k": DEFAULT_LIMIT_K,
+        "default_similarity_metric": DEFAULT_SIMILARITY_METRIC,
+        "default_similarity_threshold": DEFAULT_SIMILARITY_THRESHOLD,
+        "allowed_metrics": ALLOWED_METRICS
+    }
+    
+    # Return health status
+    status = "healthy" if all(services.values()) else "degraded"
+    return {
+        "status": status, 
+        "services": services,
+        "retrieval_config": retrieval_config,
+        "timestamp": datetime.now().isoformat()
+    }

app/api/websocket_routes.py

@@ -0,0 +1,303 @@
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Depends, status
+from typing import List, Dict
+import logging
+from datetime import datetime
+import asyncio
+import json
+import os
+from dotenv import load_dotenv
+from app.database.mongodb import session_collection
+from app.utils.utils import get_local_time
+
+# Load environment variables
+load_dotenv()
+
+# Get WebSocket configuration from environment variables
+WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Create router
+router = APIRouter(
+    tags=["WebSocket"],
+)
+
+# Store active WebSocket connections
+class ConnectionManager:
+    def __init__(self):
+        self.active_connections: List[WebSocket] = []
+
+    async def connect(self, websocket: WebSocket):
+        await websocket.accept()
+        self.active_connections.append(websocket)
+        client_info = f"{websocket.client.host}:{websocket.client.port}" if hasattr(websocket, 'client') else "Unknown"
+        logger.info(f"New WebSocket connection from {client_info}. Total connections: {len(self.active_connections)}")
+
+    def disconnect(self, websocket: WebSocket):
+        self.active_connections.remove(websocket)
+        logger.info(f"WebSocket connection removed. Total connections: {len(self.active_connections)}")
+
+    async def broadcast(self, message: Dict):
+        if not self.active_connections:
+            logger.warning("No active WebSocket connections to broadcast to")
+            return
+            
+        disconnected = []
+        for connection in self.active_connections:
+            try:
+                await connection.send_json(message)
+                logger.info(f"Message sent to WebSocket connection")
+            except Exception as e:
+                logger.error(f"Error sending message to WebSocket: {e}")
+                disconnected.append(connection)
+                
+        # Remove disconnected connections
+        for conn in disconnected:
+            if conn in self.active_connections:
+                self.active_connections.remove(conn)
+                logger.info(f"Removed disconnected WebSocket. Remaining: {len(self.active_connections)}")
+
+# Initialize connection manager
+manager = ConnectionManager()
+
+# Create full URL of WebSocket server from environment variables
+def get_full_websocket_url(server_side=False):
+    if server_side:
+        # Relative URL (for server side)
+        return WEBSOCKET_PATH
+    else:
+        # Full URL (for client)
+        # Check if should use wss:// for HTTPS
+        is_https = True if int(WEBSOCKET_PORT) == 443 else False
+        protocol = "wss" if is_https else "ws"
+        
+        # If using default port for protocol, don't include in URL
+        if (is_https and int(WEBSOCKET_PORT) == 443) or (not is_https and int(WEBSOCKET_PORT) == 80):
+            return f"{protocol}://{WEBSOCKET_SERVER}{WEBSOCKET_PATH}"
+        else:
+            return f"{protocol}://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+
+# Add GET endpoint to display WebSocket information in Swagger
+@router.get("/notify", 
+    summary="WebSocket notifications for Admin Bot",
+    description=f"""
+    This is documentation for the WebSocket endpoint.
+    
+    To connect to WebSocket:
+    1. Use the path `{get_full_websocket_url()}`
+    2. Connect using a WebSocket client library
+    3. When there are new sessions requiring attention, you will receive notifications through this connection
+    
+    Notifications are sent when:
+    - Session response starts with "I'm sorry"
+    - The system cannot answer the user's question
+    
+    Make sure to send a "keepalive" message every 5 minutes to maintain the connection.
+    """,
+    status_code=status.HTTP_200_OK
+)
+async def websocket_documentation():
+    """
+    Provides information about how to use the WebSocket endpoint /notify.
+    This endpoint is for documentation purposes only. To use WebSocket, please connect to the WebSocket URL.
+    """
+    ws_url = get_full_websocket_url()
+    return {
+        "websocket_endpoint": WEBSOCKET_PATH,
+        "connection_type": "WebSocket",
+        "protocol": "ws://",
+        "server": WEBSOCKET_SERVER,
+        "port": WEBSOCKET_PORT,
+        "full_url": ws_url,
+        "description": "Endpoint to receive notifications about new sessions requiring attention",
+        "notification_format": {
+            "type": "sorry_response",
+            "timestamp": "YYYY-MM-DD HH:MM:SS",
+            "data": {
+                "session_id": "session id",
+                "factor": "user",
+                "action": "action type",
+                "message": "User question",
+                "response": "I'm sorry...",
+                "user_id": "user id",
+                "first_name": "user's first name",
+                "last_name": "user's last name",
+                "username": "username",
+                "created_at": "creation time"
+            }
+        },
+        "client_example": """
+        import websocket
+        import json
+        import os
+        import time
+        import threading
+        from dotenv import load_dotenv
+        
+        # Load environment variables
+        load_dotenv()
+        
+        # Get WebSocket configuration from environment variables
+        WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+        WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+        WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+        
+        # Create full URL
+        ws_url = f"ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+        
+        # If using HTTPS, replace ws:// with wss://
+        # ws_url = f"wss://{WEBSOCKET_SERVER}{WEBSOCKET_PATH}"
+        
+        # Send keepalive periodically
+        def send_keepalive(ws):
+            while True:
+                try:
+                    if ws.sock and ws.sock.connected:
+                        ws.send("keepalive")
+                        print("Sent keepalive message")
+                    time.sleep(300)  # 5 minutes
+                except Exception as e:
+                    print(f"Error sending keepalive: {e}")
+                    time.sleep(60)
+        
+        def on_message(ws, message):
+            try:
+                data = json.loads(message)
+                print(f"Received notification: {data}")
+                # Process notification, e.g.: send to Telegram Admin
+                if data.get("type") == "sorry_response":
+                    session_data = data.get("data", {})
+                    user_question = session_data.get("message", "")
+                    user_name = session_data.get("first_name", "Unknown User")
+                    print(f"User {user_name} asked: {user_question}")
+                    # Code to send message to Telegram Admin
+            except json.JSONDecodeError:
+                print(f"Received non-JSON message: {message}")
+            except Exception as e:
+                print(f"Error processing message: {e}")
+        
+        def on_error(ws, error):
+            print(f"WebSocket error: {error}")
+        
+        def on_close(ws, close_status_code, close_msg):
+            print(f"WebSocket connection closed: code={close_status_code}, message={close_msg}")
+        
+        def on_open(ws):
+            print(f"WebSocket connection opened to {ws_url}")
+            # Send keepalive messages periodically in a separate thread
+            keepalive_thread = threading.Thread(target=send_keepalive, args=(ws,), daemon=True)
+            keepalive_thread.start()
+        
+        def run_forever_with_reconnect():
+            while True:
+                try:
+                    # Connect WebSocket with ping to maintain connection
+                    ws = websocket.WebSocketApp(
+                        ws_url,
+                        on_open=on_open,
+                        on_message=on_message,
+                        on_error=on_error,
+                        on_close=on_close
+                    )
+                    ws.run_forever(ping_interval=60, ping_timeout=30)
+                    print("WebSocket connection lost, reconnecting in 5 seconds...")
+                    time.sleep(5)
+                except Exception as e:
+                    print(f"WebSocket connection error: {e}")
+                    time.sleep(5)
+        
+        # Start WebSocket client in a separate thread
+        websocket_thread = threading.Thread(target=run_forever_with_reconnect, daemon=True)
+        websocket_thread.start()
+        
+        # Keep the program running
+        try:
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            print("Stopping WebSocket client...")
+        """
+    }
+
+@router.websocket("/notify")
+async def websocket_endpoint(websocket: WebSocket):
+    """
+    WebSocket endpoint to receive notifications about new sessions.
+    Admin Bot will connect to this endpoint to receive notifications when there are new sessions requiring attention.
+    """
+    await manager.connect(websocket)
+    try:
+        while True:
+            # Maintain WebSocket connection
+            data = await websocket.receive_text()
+            # Echo back to keep connection active
+            await websocket.send_json({"status": "connected", "echo": data, "timestamp": datetime.now().isoformat()})
+            logger.info(f"Received message from WebSocket: {data}")
+    except WebSocketDisconnect:
+        logger.info("WebSocket client disconnected")
+        manager.disconnect(websocket)
+    except Exception as e:
+        logger.error(f"WebSocket error: {e}")
+        manager.disconnect(websocket)
+
+# Function to send notifications over WebSocket
+async def send_notification(data: dict):
+    """
+    Send notification to all active WebSocket connections.
+    
+    This function is used to notify admin bots about new issues or questions that need attention.
+    It's triggered when the system cannot answer a user's question (response starts with "I'm sorry").
+    
+    Args:
+        data: The data to send as notification
+    """
+    try:
+        # Log number of active connections and notification attempt
+        logger.info(f"Attempting to send notification. Active connections: {len(manager.active_connections)}")
+        logger.info(f"Notification data: session_id={data.get('session_id')}, user_id={data.get('user_id')}")
+        logger.info(f"Response: {data.get('response', '')[:50]}...")
+        
+        # Check if the response starts with "I'm sorry"
+        response = data.get('response', '')
+        if not response or not isinstance(response, str):
+            logger.warning(f"Invalid response format in notification data: {response}")
+            return
+            
+        if not response.strip().lower().startswith("i'm sorry"):
+            logger.info(f"Response doesn't start with 'I'm sorry', notification not needed: {response[:50]}...")
+            return
+            
+        logger.info(f"Response starts with 'I'm sorry', sending notification")
+        
+        # Format the notification data for admin - format theo chuẩn Admin_bot
+        notification_data = {
+            "type": "sorry_response",  # Đổi type thành sorry_response để phù hợp với Admin_bot
+            "timestamp": get_local_time(),
+            "user_id": data.get('user_id', 'unknown'),
+            "message": data.get('message', ''),
+            "response": response,
+            "session_id": data.get('session_id', 'unknown'),
+            "user_info": {
+                "first_name": data.get('first_name', 'User'),
+                "last_name": data.get('last_name', ''),
+                "username": data.get('username', '')
+            }
+        }
+        
+        # Check if there are active connections
+        if not manager.active_connections:
+            logger.warning("No active WebSocket connections for notification broadcast")
+            return
+        
+        # Broadcast notification to all active connections
+        logger.info(f"Broadcasting notification to {len(manager.active_connections)} connections")
+        await manager.broadcast(notification_data)
+        logger.info("Notification broadcast completed successfully")
+        
+    except Exception as e:
+        logger.error(f"Error sending notification: {e}")
+        import traceback
+        logger.error(traceback.format_exc())

app/database/__init__.py

@@ -0,0 +1 @@
+# Database connections package 

app/database/models.py

@@ -0,0 +1,204 @@
+from sqlalchemy import Column, Integer, String, DateTime, Boolean, ForeignKey, Float, Text, LargeBinary, JSON
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship
+from .postgresql import Base
+import datetime
+
+class FAQItem(Base):
+    __tablename__ = "faq_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    question = Column(String, nullable=False)
+    answer = Column(String, nullable=False)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class EmergencyItem(Base):
+    __tablename__ = "emergency_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    phone_number = Column(String, nullable=False)
+    description = Column(String, nullable=True)
+    address = Column(String, nullable=True)
+    location = Column(String, nullable=True)  # Will be converted to/from PostGIS POINT type
+    priority = Column(Integer, default=0)
+    is_active = Column(Boolean, default=True)
+    section = Column(String, nullable=True)  # Section field (16.1, 16.2.1, 16.2.2, 16.3)
+    section_id = Column(Integer, nullable=True)  # Numeric identifier for section
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class EventItem(Base):
+    __tablename__ = "event_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    description = Column(Text, nullable=False)
+    address = Column(String, nullable=False)
+    location = Column(String, nullable=True)  # Will be converted to/from PostGIS POINT type
+    date_start = Column(DateTime, nullable=False)
+    date_end = Column(DateTime, nullable=True)
+    price = Column(JSON, nullable=True)
+    url = Column(String, nullable=True)
+    is_active = Column(Boolean, default=True)
+    featured = Column(Boolean, default=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class AboutPixity(Base):
+    __tablename__ = "about_pixity"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    content = Column(Text, nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class SolanaSummit(Base):
+    __tablename__ = "solana_summit"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    content = Column(Text, nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class DaNangBucketList(Base):
+    __tablename__ = "danang_bucket_list"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    content = Column(Text, nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class VectorDatabase(Base):
+    __tablename__ = "vector_database"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False, unique=True)
+    description = Column(String, nullable=True)
+    pinecone_index = Column(String, nullable=False)
+    api_key_id = Column(Integer, ForeignKey("api_key.id"), nullable=True)
+    status = Column(String, default="active")
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    documents = relationship("Document", back_populates="vector_database")
+    vector_statuses = relationship("VectorStatus", back_populates="vector_database")
+    engine_associations = relationship("EngineVectorDb", back_populates="vector_database")
+    api_key_ref = relationship("ApiKey", foreign_keys=[api_key_id])
+
+class Document(Base):
+    __tablename__ = "document"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    file_type = Column(String, nullable=True)
+    content_type = Column(String, nullable=True)
+    size = Column(Integer, nullable=True)
+    is_embedded = Column(Boolean, default=False)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    vector_database = relationship("VectorDatabase", back_populates="documents")
+    vector_statuses = relationship("VectorStatus", back_populates="document")
+    file_content_ref = relationship("DocumentContent", back_populates="document", uselist=False, cascade="all, delete-orphan")
+
+class DocumentContent(Base):
+    __tablename__ = "document_content"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    document_id = Column(Integer, ForeignKey("document.id"), nullable=False, unique=True)
+    file_content = Column(LargeBinary, nullable=True)
+    created_at = Column(DateTime, server_default=func.now())
+    
+    # Relationships
+    document = relationship("Document", back_populates="file_content_ref")
+
+class VectorStatus(Base):
+    __tablename__ = "vector_status"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    document_id = Column(Integer, ForeignKey("document.id"), nullable=False)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    vector_id = Column(String, nullable=True)
+    status = Column(String, default="pending")
+    error_message = Column(String, nullable=True)
+    embedded_at = Column(DateTime, nullable=True)
+    
+    # Relationships
+    document = relationship("Document", back_populates="vector_statuses")
+    vector_database = relationship("VectorDatabase", back_populates="vector_statuses")
+
+class TelegramBot(Base):
+    __tablename__ = "telegram_bot"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    username = Column(String, nullable=False, unique=True)
+    token = Column(String, nullable=False)
+    status = Column(String, default="inactive")
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    bot_engines = relationship("BotEngine", back_populates="bot")
+
+class ChatEngine(Base):
+    __tablename__ = "chat_engine"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    answer_model = Column(String, nullable=False)
+    system_prompt = Column(Text, nullable=True)
+    empty_response = Column(String, nullable=True)
+    similarity_top_k = Column(Integer, default=3)
+    vector_distance_threshold = Column(Float, default=0.75)
+    grounding_threshold = Column(Float, default=0.2)
+    use_public_information = Column(Boolean, default=False)
+    status = Column(String, default="active")
+    created_at = Column(DateTime, server_default=func.now())
+    last_modified = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    bot_engines = relationship("BotEngine", back_populates="engine")
+    engine_vector_dbs = relationship("EngineVectorDb", back_populates="engine")
+
+class BotEngine(Base):
+    __tablename__ = "bot_engine"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    bot_id = Column(Integer, ForeignKey("telegram_bot.id"), nullable=False)
+    engine_id = Column(Integer, ForeignKey("chat_engine.id"), nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    
+    # Relationships
+    bot = relationship("TelegramBot", back_populates="bot_engines")
+    engine = relationship("ChatEngine", back_populates="bot_engines")
+
+class EngineVectorDb(Base):
+    __tablename__ = "engine_vector_db"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    engine_id = Column(Integer, ForeignKey("chat_engine.id"), nullable=False)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    priority = Column(Integer, default=0)
+    
+    # Relationships
+    engine = relationship("ChatEngine", back_populates="engine_vector_dbs")
+    vector_database = relationship("VectorDatabase", back_populates="engine_associations")
+
+class ApiKey(Base):
+    __tablename__ = "api_key"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    key_type = Column(String, nullable=False)
+    key_value = Column(Text, nullable=False)
+    description = Column(Text, nullable=True)
+    created_at = Column(DateTime, server_default=func.now())
+    last_used = Column(DateTime, nullable=True)
+    expires_at = Column(DateTime, nullable=True)
+    is_active = Column(Boolean, default=True) 

app/database/mongodb.py

@@ -0,0 +1,221 @@
+import os
+from pymongo import MongoClient
+from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError
+from dotenv import load_dotenv
+from datetime import datetime, timedelta
+import pytz
+import logging
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# MongoDB connection string from .env
+MONGODB_URL = os.getenv("MONGODB_URL")
+DB_NAME = os.getenv("DB_NAME", "Telegram")
+COLLECTION_NAME = os.getenv("COLLECTION_NAME", "session_chat")
+
+# Set timeout for MongoDB connection
+MONGODB_TIMEOUT = int(os.getenv("MONGODB_TIMEOUT", "5000"))  # 5 seconds by default
+
+# Legacy cache settings - now only used for configuration purposes
+HISTORY_CACHE_TTL = int(os.getenv("HISTORY_CACHE_TTL", "3600"))  # 1 hour by default
+HISTORY_QUEUE_SIZE = int(os.getenv("HISTORY_QUEUE_SIZE", "10"))  # 10 items by default
+
+# Create MongoDB connection with timeout
+try:
+    client = MongoClient(MONGODB_URL, serverSelectionTimeoutMS=MONGODB_TIMEOUT)
+    db = client[DB_NAME]
+
+    # Collections
+    session_collection = db[COLLECTION_NAME]
+    logger.info(f"MongoDB connection initialized to {DB_NAME}.{COLLECTION_NAME}")
+    
+except Exception as e:
+    logger.error(f"Failed to initialize MongoDB connection: {e}")
+    # Don't raise exception to avoid crash during startup, error handling will be done in functions
+
+# Check MongoDB connection
+def check_db_connection():
+    """Check MongoDB connection"""
+    try:
+        # Issue a ping to confirm a successful connection
+        client.admin.command('ping')
+        logger.info("MongoDB connection is working")
+        return True
+    except (ConnectionFailure, ServerSelectionTimeoutError) as e:
+        logger.error(f"MongoDB connection failed: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unknown error when checking MongoDB connection: {e}")
+        return False
+
+# Timezone for Asia/Ho_Chi_Minh
+asia_tz = pytz.timezone('Asia/Ho_Chi_Minh')
+
+def get_local_time():
+    """Get current time in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz).strftime("%Y-%m-%d %H:%M:%S")
+
+def get_local_datetime():
+    """Get current datetime object in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz)
+
+# For backward compatibility
+get_vietnam_time = get_local_time
+get_vietnam_datetime = get_local_datetime
+
+# Utility functions
+def save_session(session_id, factor, action, first_name, last_name, message, user_id, username, response=None):
+    """Save user session to MongoDB"""
+    try:
+        session_data = {
+            "session_id": session_id,
+            "factor": factor,
+            "action": action,
+            "created_at": get_local_time(),
+            "created_at_datetime": get_local_datetime(),
+            "first_name": first_name,
+            "last_name": last_name,
+            "message": message,
+            "user_id": user_id,
+            "username": username,
+            "response": response
+        }
+        result = session_collection.insert_one(session_data)
+        logger.info(f"Session saved with ID: {result.inserted_id}")
+        
+        return {
+            "acknowledged": result.acknowledged,
+            "inserted_id": str(result.inserted_id),
+            "session_data": session_data
+        }
+    except Exception as e:
+        logger.error(f"Error saving session: {e}")
+        raise
+
+def update_session_response(session_id, response):
+    """Update a session with response"""
+    try:
+        # Lấy session hiện có
+        existing_session = session_collection.find_one({"session_id": session_id})
+        
+        if not existing_session:
+            logger.warning(f"No session found with ID: {session_id}")
+            return False
+        
+        result = session_collection.update_one(
+            {"session_id": session_id},
+            {"$set": {"response": response}}
+        )
+        
+        logger.info(f"Session {session_id} updated with response")
+        return True
+    except Exception as e:
+        logger.error(f"Error updating session response: {e}")
+        raise
+
+def get_recent_sessions(user_id, action, n=3):
+    """Get n most recent sessions for a specific user and action"""
+    try:
+        # Truy vấn trực tiếp từ MongoDB
+        result = list(
+            session_collection.find(
+                {"user_id": user_id, "action": action},
+                {"_id": 0, "message": 1, "response": 1}
+            ).sort("created_at_datetime", -1).limit(n)
+        )
+        
+        logger.debug(f"Retrieved {len(result)} recent sessions for user {user_id}, action {action}")
+        return result
+    except Exception as e:
+        logger.error(f"Error getting recent sessions: {e}")
+        return []
+
+def get_chat_history(user_id, n = 5) -> str:
+    """
+    Lấy lịch sử chat cho user_id từ MongoDB và ghép thành chuỗi theo định dạng:
+    
+    User: ...
+    Bot: ...
+    User: ...
+    Bot: ...
+    
+    Chỉ lấy history sau lệnh /start hoặc /clear mới nhất
+    """
+    try:
+        # Tìm session /start hoặc /clear mới nhất
+        reset_session = session_collection.find_one(
+            {
+                "user_id": str(user_id), 
+                "$or": [
+                    {"action": "start"},
+                    {"action": "clear"}
+                ]
+            },
+            sort=[("created_at_datetime", -1)]
+        )
+        
+        # Nếu không tìm thấy session reset nào, lấy n session gần nhất
+        if reset_session:
+            reset_time = reset_session["created_at_datetime"]
+            # Lấy các session sau reset_time
+            docs = list(
+                session_collection.find({
+                    "user_id": str(user_id),
+                    "created_at_datetime": {"$gt": reset_time}
+                }).sort("created_at_datetime", 1)
+            )
+            logger.info(f"Lấy {len(docs)} session sau lệnh {reset_session['action']} lúc {reset_time}")
+        else:
+            # Không tìm thấy reset session, lấy n session gần nhất
+            docs = list(session_collection.find({"user_id": str(user_id)}).sort("created_at", -1).limit(n))
+            # Đảo ngược để có thứ tự từ cũ đến mới
+            docs.reverse()
+            logger.info(f"Không tìm thấy session reset, lấy {len(docs)} session gần nhất")
+            
+        if not docs:
+            logger.info(f"Không tìm thấy dữ liệu cho user_id: {user_id}")
+            return ""
+        
+        conversation_lines = []
+        # Xử lý từng document theo cấu trúc mới
+        for doc in docs:
+            factor = doc.get("factor", "").lower()
+            action = doc.get("action", "").lower()
+            message = doc.get("message", "")
+            response = doc.get("response", "")
+            
+            # Bỏ qua lệnh start và clear
+            if action in ["start", "clear"]:
+                continue
+                
+            if factor == "user" and action == "asking_freely":
+                conversation_lines.append(f"User: {message}")
+                conversation_lines.append(f"Bot: {response}")
+        
+        # Ghép các dòng thành chuỗi
+        return "\n".join(conversation_lines)
+    except Exception as e:
+        logger.error(f"Lỗi khi lấy lịch sử chat cho user_id {user_id}: {e}")
+        return ""
+
+def get_request_history(user_id, n=3):
+    """Get the most recent user requests to use as context for retrieval"""
+    try:
+        # Truy vấn trực tiếp từ MongoDB
+        history = get_chat_history(user_id, n)
+        
+        # Just extract the questions for context
+        requests = []
+        for line in history.split('\n'):
+            if line.startswith("User: "):
+                requests.append(line[6:])  # Lấy nội dung sau "User: "
+            
+        # Join all recent requests into a single string for context
+        return " ".join(requests)
+    except Exception as e:
+        logger.error(f"Error getting request history: {e}")
+        return "" 

app/database/pinecone.py

@@ -0,0 +1,573 @@
+import os
+from pinecone import Pinecone
+from dotenv import load_dotenv
+import logging
+from typing import Optional, List, Dict, Any, Union, Tuple
+import time
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+import google.generativeai as genai
+from langchain_core.retrievers import BaseRetriever
+from langchain.callbacks.manager import Callbacks
+from langchain_core.documents import Document
+from langchain_core.pydantic_v1 import Field
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# Pinecone API key and index name
+PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
+PINECONE_INDEX_NAME = os.getenv("PINECONE_INDEX_NAME")
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+
+# Pinecone retrieval configuration
+DEFAULT_LIMIT_K = int(os.getenv("PINECONE_DEFAULT_LIMIT_K", "10"))
+DEFAULT_TOP_K = int(os.getenv("PINECONE_DEFAULT_TOP_K", "6"))
+DEFAULT_SIMILARITY_METRIC = os.getenv("PINECONE_DEFAULT_SIMILARITY_METRIC", "cosine")
+DEFAULT_SIMILARITY_THRESHOLD = float(os.getenv("PINECONE_DEFAULT_SIMILARITY_THRESHOLD", "0.75"))
+ALLOWED_METRICS = os.getenv("PINECONE_ALLOWED_METRICS", "cosine,dotproduct,euclidean").split(",")
+
+# Export constants for importing elsewhere
+__all__ = [
+    'get_pinecone_index', 
+    'check_db_connection', 
+    'search_vectors', 
+    'upsert_vectors', 
+    'delete_vectors', 
+    'fetch_metadata',
+    'get_chain',
+    'DEFAULT_TOP_K',
+    'DEFAULT_LIMIT_K',
+    'DEFAULT_SIMILARITY_METRIC',
+    'DEFAULT_SIMILARITY_THRESHOLD',
+    'ALLOWED_METRICS',
+    'ThresholdRetriever'
+]
+
+# Configure Google API
+if GOOGLE_API_KEY:
+    genai.configure(api_key=GOOGLE_API_KEY)
+
+# Initialize global variables to store instances of Pinecone and index
+pc = None
+index = None
+_retriever_instance = None
+
+# Check environment variables
+if not PINECONE_API_KEY:
+    logger.error("PINECONE_API_KEY is not set in environment variables")
+
+if not PINECONE_INDEX_NAME:
+    logger.error("PINECONE_INDEX_NAME is not set in environment variables")
+
+# Initialize Pinecone
+def init_pinecone():
+    """Initialize pinecone connection using new API"""
+    global pc, index
+    
+    try:
+        # Only initialize if not already initialized
+        if pc is None:
+            logger.info(f"Initializing Pinecone connection to index {PINECONE_INDEX_NAME}...")
+            
+            # Check if API key and index name are set
+            if not PINECONE_API_KEY:
+                logger.error("PINECONE_API_KEY is not set in environment variables")
+                return None
+                
+            if not PINECONE_INDEX_NAME:
+                logger.error("PINECONE_INDEX_NAME is not set in environment variables")
+                return None
+            
+            # Initialize Pinecone client using the new API
+            pc = Pinecone(api_key=PINECONE_API_KEY)
+            
+            try:
+                # Check if index exists
+                index_list = pc.list_indexes()
+                
+                if not hasattr(index_list, 'names') or PINECONE_INDEX_NAME not in index_list.names():
+                    logger.error(f"Index {PINECONE_INDEX_NAME} does not exist in Pinecone")
+                    return None
+                
+                # Get existing index
+                index = pc.Index(PINECONE_INDEX_NAME)
+                logger.info(f"Pinecone connection established to index {PINECONE_INDEX_NAME}")
+            except Exception as connection_error:
+                logger.error(f"Error connecting to Pinecone index: {connection_error}")
+                return None
+            
+        return index
+    except ImportError as e:
+        logger.error(f"Required package for Pinecone is missing: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unexpected error initializing Pinecone: {e}")
+        return None
+
+# Get Pinecone index singleton
+def get_pinecone_index():
+    """Get Pinecone index"""
+    global index
+    if index is None:
+        index = init_pinecone()
+    return index
+
+# Check Pinecone connection
+def check_db_connection():
+    """Check Pinecone connection"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            return False
+            
+        # Check index information to confirm connection is working
+        stats = pinecone_index.describe_index_stats()
+        
+        # Get total vector count from the new result structure
+        total_vectors = stats.get('total_vector_count', 0)
+        if hasattr(stats, 'namespaces'):
+            # If there are namespaces, calculate total vector count from namespaces
+            total_vectors = sum(ns.get('vector_count', 0) for ns in stats.namespaces.values())
+            
+        logger.info(f"Pinecone connection is working. Total vectors: {total_vectors}")
+        return True
+    except Exception as e:
+        logger.error(f"Error in Pinecone connection: {e}")
+        return False
+
+# Convert similarity score based on the metric
+def convert_score(score: float, metric: str) -> float:
+    """
+    Convert similarity score to a 0-1 scale based on the metric used.
+    For metrics like euclidean distance where lower is better, we invert the score.
+    
+    Args:
+        score: The raw similarity score
+        metric: The similarity metric used
+        
+    Returns:
+        A normalized score between 0-1 where higher means more similar
+    """
+    if metric.lower() in ["euclidean", "l2"]:
+        # For distance metrics (lower is better), we inverse and normalize
+        # Assuming max reasonable distance is 2.0 for normalized vectors
+        return max(0, 1 - (score / 2.0))
+    else:
+        # For cosine and dot product (higher is better), return as is
+        return score
+
+# Filter results based on similarity threshold
+def filter_by_threshold(results, threshold: float, metric: str) -> List[Dict]:
+    """
+    Filter query results based on similarity threshold.
+    
+    Args:
+        results: The query results from Pinecone
+        threshold: The similarity threshold (0-1)
+        metric: The similarity metric used
+        
+    Returns:
+        Filtered list of matches
+    """
+    filtered_matches = []
+    
+    if not hasattr(results, 'matches'):
+        return filtered_matches
+        
+    for match in results.matches:
+        # Get the score
+        score = getattr(match, 'score', 0)
+        
+        # Convert score based on metric
+        normalized_score = convert_score(score, metric)
+        
+        # Filter based on threshold
+        if normalized_score >= threshold:
+            # Add normalized score as an additional attribute
+            match.normalized_score = normalized_score
+            filtered_matches.append(match)
+    
+    return filtered_matches
+
+# Search vectors in Pinecone with advanced options
+async def search_vectors(
+    query_vector, 
+    top_k: int = DEFAULT_TOP_K,
+    limit_k: int = DEFAULT_LIMIT_K,
+    similarity_metric: str = DEFAULT_SIMILARITY_METRIC,
+    similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+    namespace: str = "Default", 
+    filter: Optional[Dict] = None
+) -> Dict:
+    """
+    Search for most similar vectors in Pinecone with advanced filtering options.
+    
+    Args:
+        query_vector: The query vector
+        top_k: Number of results to return (after threshold filtering)
+        limit_k: Maximum number of results to retrieve from Pinecone
+        similarity_metric: Similarity metric to use (cosine, dotproduct, euclidean)
+        similarity_threshold: Threshold for similarity (0-1)
+        namespace: Namespace to search in
+        filter: Filter query
+        
+    Returns:
+        Search results with matches filtered by threshold
+    """
+    try:
+        # Validate parameters
+        if similarity_metric not in ALLOWED_METRICS:
+            logger.warning(f"Invalid similarity metric: {similarity_metric}. Using default: {DEFAULT_SIMILARITY_METRIC}")
+            similarity_metric = DEFAULT_SIMILARITY_METRIC
+            
+        if limit_k < top_k:
+            logger.warning(f"limit_k ({limit_k}) must be greater than or equal to top_k ({top_k}). Setting limit_k to {top_k}")
+            limit_k = top_k
+        
+        # Perform search directly without cache
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for search")
+            return None
+            
+        # Query Pinecone with the provided metric and higher limit_k to allow for threshold filtering
+        results = pinecone_index.query(
+            vector=query_vector,
+            top_k=limit_k,  # Retrieve more results than needed to allow for threshold filtering
+            namespace=namespace,
+            filter=filter,
+            include_metadata=True,
+            include_values=False,  # No need to return vector values to save bandwidth
+            metric=similarity_metric  # Specify similarity metric
+        )
+        
+        # Filter results by threshold
+        filtered_matches = filter_by_threshold(results, similarity_threshold, similarity_metric)
+        
+        # Limit to top_k after filtering
+        filtered_matches = filtered_matches[:top_k]
+        
+        # Create a new results object with filtered matches
+        results.matches = filtered_matches
+        
+        # Log search result metrics
+        match_count = len(filtered_matches)
+        logger.info(f"Pinecone search returned {match_count} matches after threshold filtering (metric: {similarity_metric}, threshold: {similarity_threshold}, namespace: {namespace})")
+        
+        return results
+    except Exception as e:
+        logger.error(f"Error searching vectors: {e}")
+        return None
+
+# Upsert vectors to Pinecone
+async def upsert_vectors(vectors, namespace="Default"):
+    """Upsert vectors to Pinecone index"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for upsert")
+            return None
+            
+        response = pinecone_index.upsert(
+            vectors=vectors,
+            namespace=namespace
+        )
+        
+        # Log upsert metrics
+        upserted_count = response.get('upserted_count', 0)
+        logger.info(f"Upserted {upserted_count} vectors to Pinecone")
+        
+        return response
+    except Exception as e:
+        logger.error(f"Error upserting vectors: {e}")
+        return None
+
+# Delete vectors from Pinecone
+async def delete_vectors(ids, namespace="Default"):
+    """Delete vectors from Pinecone index"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for delete")
+            return False
+            
+        response = pinecone_index.delete(
+            ids=ids,
+            namespace=namespace
+        )
+        
+        logger.info(f"Deleted vectors with IDs {ids} from Pinecone")
+        return True
+    except Exception as e:
+        logger.error(f"Error deleting vectors: {e}")
+        return False
+
+# Fetch vector metadata from Pinecone
+async def fetch_metadata(ids, namespace="Default"):
+    """Fetch metadata for specific vector IDs"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for fetch")
+            return None
+            
+        response = pinecone_index.fetch(
+            ids=ids,
+            namespace=namespace
+        )
+        
+        return response
+    except Exception as e:
+        logger.error(f"Error fetching vector metadata: {e}")
+        return None
+
+# Create a custom retriever class for Langchain integration
+class ThresholdRetriever(BaseRetriever):
+    """
+    Custom retriever that supports threshold-based filtering and multiple similarity metrics.
+    This integrates with the Langchain ecosystem while using our advanced retrieval logic.
+    """
+    
+    vectorstore: Any = Field(description="Vector store to use for retrieval")
+    embeddings: Any = Field(description="Embeddings model to use for retrieval")
+    search_kwargs: Dict[str, Any] = Field(default_factory=dict, description="Search kwargs for the vectorstore")
+    top_k: int = Field(default=DEFAULT_TOP_K, description="Number of results to return after filtering")
+    limit_k: int = Field(default=DEFAULT_LIMIT_K, description="Maximum number of results to retrieve from Pinecone")
+    similarity_metric: str = Field(default=DEFAULT_SIMILARITY_METRIC, description="Similarity metric to use")
+    similarity_threshold: float = Field(default=DEFAULT_SIMILARITY_THRESHOLD, description="Threshold for similarity")
+    namespace: str = "Default"
+
+    class Config:
+        """Configuration for this pydantic object."""
+        arbitrary_types_allowed = True
+    
+    async def search_vectors_sync(
+        self, query_vector, 
+        top_k: int = DEFAULT_TOP_K,
+        limit_k: int = DEFAULT_LIMIT_K,
+        similarity_metric: str = DEFAULT_SIMILARITY_METRIC,
+        similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+        namespace: str = "Default", 
+        filter: Optional[Dict] = None
+    ) -> Dict:
+        """Synchronous wrapper for search_vectors"""
+        import asyncio
+        try:
+            # Get current event loop or create a new one
+            try:
+                loop = asyncio.get_event_loop()
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            
+            # Use event loop to run async function
+            if loop.is_running():
+                # If we're in an event loop, use asyncio.create_task
+                task = asyncio.create_task(search_vectors(
+                    query_vector=query_vector,
+                    top_k=top_k,
+                    limit_k=limit_k,
+                    similarity_metric=similarity_metric,
+                    similarity_threshold=similarity_threshold,
+                    namespace=namespace,
+                    filter=filter
+                ))
+                return await task
+            else:
+                # If not in an event loop, just await directly
+                return await search_vectors(
+                    query_vector=query_vector,
+                    top_k=top_k,
+                    limit_k=limit_k,
+                    similarity_metric=similarity_metric,
+                    similarity_threshold=similarity_threshold,
+                    namespace=namespace,
+                    filter=filter
+                )
+        except Exception as e:
+            logger.error(f"Error in search_vectors_sync: {e}")
+            return None
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: Callbacks = None
+    ) -> List[Document]:
+        """
+        Get documents relevant to the query using threshold-based retrieval.
+        
+        Args:
+            query: The query string
+            run_manager: The callbacks manager
+            
+        Returns:
+            List of relevant documents
+        """
+        # Generate embedding for query using the embeddings model
+        try:
+            # Use the embeddings model we stored in the class
+            embedding = self.embeddings.embed_query(query)
+        except Exception as e:
+            logger.error(f"Error generating embedding: {e}")
+            # Fallback to creating a new embedding model if needed
+            embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+            embedding = embedding_model.embed_query(query)
+        
+        # Perform search with advanced options - avoid asyncio.run()
+        import asyncio
+        
+        # Get or create event loop
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        # Run asynchronous search in a safe way
+        if loop.is_running():
+            # We're inside an existing event loop (like in FastAPI)
+            # Use a different approach - convert it to a synchronous call
+            from concurrent.futures import ThreadPoolExecutor
+            import functools
+            
+            # Define a wrapper function to run in a thread
+            def run_async_in_thread():
+                # Create a new event loop for this thread
+                thread_loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(thread_loop)
+                # Run the coroutine and return the result
+                return thread_loop.run_until_complete(search_vectors(
+                    query_vector=embedding,
+                    top_k=self.top_k,
+                    limit_k=self.limit_k,
+                    similarity_metric=self.similarity_metric,
+                    similarity_threshold=self.similarity_threshold,
+                    namespace=self.namespace,
+                    # filter=self.search_kwargs.get("filter", None)
+                ))
+            
+            # Run the async function in a thread
+            with ThreadPoolExecutor() as executor:
+                search_result = executor.submit(run_async_in_thread).result()
+        else:
+            # No event loop running, we can use run_until_complete
+            search_result = loop.run_until_complete(search_vectors(
+                query_vector=embedding,
+                top_k=self.top_k,
+                limit_k=self.limit_k,
+                similarity_metric=self.similarity_metric,
+                similarity_threshold=self.similarity_threshold,
+                namespace=self.namespace,
+                # filter=self.search_kwargs.get("filter", None)
+            ))
+        
+        # Convert to documents
+        documents = []
+        if search_result and hasattr(search_result, 'matches'):
+            for match in search_result.matches:
+                # Extract metadata
+                metadata = {}
+                if hasattr(match, 'metadata'):
+                    metadata = match.metadata
+                
+                # Add score to metadata
+                score = getattr(match, 'score', 0)
+                normalized_score = getattr(match, 'normalized_score', score)
+                metadata['score'] = score
+                metadata['normalized_score'] = normalized_score
+                
+                # Extract text
+                text = metadata.get('text', '')
+                if 'text' in metadata:
+                    del metadata['text']  # Remove from metadata since it's the content
+                
+                # Create Document
+                doc = Document(
+                    page_content=text,
+                    metadata=metadata
+                )
+                documents.append(doc)
+        
+        return documents
+
+# Get the retrieval chain with Pinecone vector store
+def get_chain(
+    index_name=PINECONE_INDEX_NAME, 
+    namespace="Default", 
+    top_k=DEFAULT_TOP_K, 
+    limit_k=DEFAULT_LIMIT_K,
+    similarity_metric=DEFAULT_SIMILARITY_METRIC, 
+    similarity_threshold=DEFAULT_SIMILARITY_THRESHOLD
+):
+    """
+    Get the retrieval chain with Pinecone vector store using threshold-based retrieval.
+    
+    Args:
+        index_name: Pinecone index name
+        namespace: Pinecone namespace
+        top_k: Number of results to return after filtering
+        limit_k: Maximum number of results to retrieve from Pinecone
+        similarity_metric: Similarity metric to use (cosine, dotproduct, euclidean)
+        similarity_threshold: Threshold for similarity (0-1)
+        
+    Returns:
+        ThresholdRetriever instance
+    """
+    global _retriever_instance
+    try:
+        # If already initialized with same parameters, return cached instance
+        if _retriever_instance is not None:
+            return _retriever_instance
+            
+        start_time = time.time()
+        logger.info("Initializing new retriever chain with threshold-based filtering")
+        
+        # Initialize embeddings model
+        embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        
+        # Get index
+        pinecone_index = get_pinecone_index()
+        if not pinecone_index:
+            logger.error("Failed to get Pinecone index for retriever chain")
+            return None
+            
+        # Get statistics for logging
+        try:
+            stats = pinecone_index.describe_index_stats()
+            total_vectors = stats.get('total_vector_count', 0)
+            logger.info(f"Pinecone index stats - Total vectors: {total_vectors}")
+        except Exception as e:
+            logger.error(f"Error getting index stats: {e}")
+        
+        # Use Pinecone from langchain_community.vectorstores
+        from langchain_community.vectorstores import Pinecone as LangchainPinecone
+        
+        logger.info(f"Creating Pinecone vectorstore with index: {index_name}, namespace: {namespace}")
+        vectorstore = LangchainPinecone.from_existing_index(
+            embedding=embeddings,
+            index_name=index_name,
+            namespace=namespace,
+            text_key="text" 
+        )
+        
+        # Create threshold-based retriever
+        logger.info(f"Creating ThresholdRetriever with top_k={top_k}, limit_k={limit_k}, " +
+                    f"metric={similarity_metric}, threshold={similarity_threshold}")
+        
+        # Create ThresholdRetriever with both vectorstore and embeddings
+        _retriever_instance = ThresholdRetriever(
+            vectorstore=vectorstore,
+            embeddings=embeddings,  # Pass embeddings separately
+            top_k=top_k,
+            limit_k=limit_k,
+            similarity_metric=similarity_metric,
+            similarity_threshold=similarity_threshold
+        )
+        
+        logger.info(f"Pinecone retriever initialized in {time.time() - start_time:.2f} seconds")
+        
+        return _retriever_instance
+    except Exception as e:
+        logger.error(f"Error creating retrieval chain: {e}")
+        return None

app/database/postgresql.py

@@ -0,0 +1,192 @@
+import os
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import SQLAlchemyError, OperationalError
+from dotenv import load_dotenv
+import logging
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# Get DB connection mode from environment
+DB_CONNECTION_MODE = os.getenv("DB_CONNECTION_MODE", "aiven")
+
+# Set connection string based on mode
+if DB_CONNECTION_MODE == "aiven":
+    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+else:
+    # Default or other connection modes can be added here
+    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+
+if not DATABASE_URL:
+    logger.error("No database URL configured. Please set AIVEN_DB_URL environment variable.")
+    DATABASE_URL = "postgresql://localhost/test"  # Fallback to avoid crash on startup
+
+# Create SQLAlchemy engine with optimized settings
+try:
+    engine = create_engine(
+        DATABASE_URL,
+        pool_size=10,  # Limit max connections
+        max_overflow=5,  # Allow temporary overflow of connections
+        pool_timeout=30,  # Timeout waiting for connection from pool
+        pool_recycle=300,  # Recycle connections every 5 minutes
+        pool_pre_ping=True,  # Verify connection is still valid before using it
+        connect_args={
+            "connect_timeout": 5,   # Connection timeout in seconds
+            "keepalives": 1,        # Enable TCP keepalives
+            "keepalives_idle": 30,  # Time before sending keepalives
+            "keepalives_interval": 10, # Time between keepalives
+            "keepalives_count": 5,  # Number of keepalive probes
+            "application_name": "pixagent_api" # Identify app in PostgreSQL logs
+        },
+        # Performance optimizations
+        isolation_level="READ COMMITTED",  # Lower isolation level for better performance
+        echo=False,                 # Disable SQL echo to reduce overhead
+        echo_pool=False,            # Disable pool logging
+        future=True,                # Use SQLAlchemy 2.0 features
+        # Execution options for common queries
+        execution_options={
+            "compiled_cache": {},   # Use an empty dict for compiled query caching
+            "logging_token": "SQL", # Tag for query logging
+        }
+    )
+    logger.info("PostgreSQL engine initialized with optimized settings")
+except Exception as e:
+    logger.error(f"Failed to initialize PostgreSQL engine: {e}")
+    # Don't raise exception to avoid crash on startup
+
+# Create optimized session factory
+SessionLocal = sessionmaker(
+    autocommit=False, 
+    autoflush=False, 
+    bind=engine,
+    expire_on_commit=False  # Prevent automatic reloading after commit
+)
+
+# Base class for declarative models - use sqlalchemy.orm for SQLAlchemy 2.0 compatibility
+from sqlalchemy.orm import declarative_base
+Base = declarative_base()
+
+# Check PostgreSQL connection
+def check_db_connection():
+    """Check PostgreSQL connection status"""
+    try:
+        # Simple query to verify connection
+        with engine.connect() as connection:
+            connection.execute(text("SELECT 1")).fetchone()
+        logger.info("PostgreSQL connection successful")
+        return True
+    except OperationalError as e:
+        logger.error(f"PostgreSQL connection failed: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unknown error checking PostgreSQL connection: {e}")
+        return False
+
+# Dependency to get DB session with improved error handling
+def get_db():
+    """Get PostgreSQL database session"""
+    db = SessionLocal()
+    try:
+        # Test connection
+        db.execute(text("SELECT 1")).fetchone()
+        yield db
+    except Exception as e:
+        logger.error(f"DB connection error: {e}")
+        raise
+    finally:
+        db.close()  # Ensure connection is closed and returned to pool
+
+# Create tables in database if they don't exist
+def create_tables():
+    """Create tables in database"""
+    try:
+        Base.metadata.create_all(bind=engine)
+        logger.info("Database tables created or already exist")
+        return True
+    except SQLAlchemyError as e:
+        logger.error(f"Failed to create database tables (SQLAlchemy error): {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to create database tables (unexpected error): {e}")
+        return False
+
+# Function to create indexes for better performance
+def create_indexes():
+    """Create indexes for better query performance"""
+    try:
+        with engine.connect() as conn:
+            try:
+                # Index for featured events - use try-except to handle if index already exists
+                conn.execute(text("""
+                    CREATE INDEX idx_event_featured 
+                    ON event_item(featured)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_event_featured already exists")
+            
+            try:
+                # Index for active events
+                conn.execute(text("""
+                    CREATE INDEX idx_event_active 
+                    ON event_item(is_active)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_event_active already exists")
+            
+            try:
+                # Index for date filtering
+                conn.execute(text("""
+                    CREATE INDEX idx_event_date_start 
+                    ON event_item(date_start)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_event_date_start already exists")
+            
+            try:
+                # Composite index for combined filtering
+                conn.execute(text("""
+                    CREATE INDEX idx_event_featured_active 
+                    ON event_item(featured, is_active)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_event_featured_active already exists")
+                
+            # Indexes for FAQ and Emergency tables
+            try:
+                # FAQ active flag index
+                conn.execute(text("""
+                    CREATE INDEX idx_faq_active 
+                    ON faq_item(is_active)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_faq_active already exists")
+            
+            try:
+                # Emergency contact active flag and priority indexes
+                conn.execute(text("""
+                    CREATE INDEX idx_emergency_active 
+                    ON emergency_item(is_active)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_emergency_active already exists")
+                
+            try:
+                conn.execute(text("""
+                    CREATE INDEX idx_emergency_priority 
+                    ON emergency_item(priority)
+                """))
+            except SQLAlchemyError:
+                logger.info("Index idx_emergency_priority already exists")
+            
+            conn.commit()
+            
+        logger.info("Database indexes created or verified")
+        return True
+    except SQLAlchemyError as e:
+        logger.error(f"Failed to create indexes: {e}")
+        return False 

app/models/__init__.py

@@ -0,0 +1 @@
+# Pydantic models package 

app/models/mongodb_models.py

@@ -0,0 +1,55 @@
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+import uuid
+
+class SessionBase(BaseModel):
+    """Base model for session data"""
+    session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    factor: str
+    action: str
+    first_name: str
+    last_name: Optional[str] = None
+    message: Optional[str] = None
+    user_id: str
+    username: Optional[str] = None
+
+class SessionCreate(SessionBase):
+    """Model for creating new session"""
+    response: Optional[str] = None
+
+class SessionResponse(SessionBase):
+    """Response model for session data"""
+    created_at: str
+    response: Optional[str] = None
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "session_id": "123e4567-e89b-12d3-a456-426614174000",
+                "factor": "user",
+                "action": "asking_freely",
+                "created_at": "2023-06-01 14:30:45",
+                "first_name": "John",
+                "last_name": "Doe",
+                "message": "How can I find emergency contacts?",
+                "user_id": "12345678",
+                "username": "johndoe",
+                "response": "You can find emergency contacts in the Emergency section..."
+            }
+        }
+    )
+
+class HistoryRequest(BaseModel):
+    """Request model for history"""
+    user_id: str
+    n: int = 3
+
+class QuestionAnswer(BaseModel):
+    """Model for question-answer pair"""
+    question: str
+    answer: str
+
+class HistoryResponse(BaseModel):
+    """Response model for history"""
+    history: List[QuestionAnswer] 

app/models/pdf_models.py

@@ -0,0 +1,52 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict, Any
+
+class PDFUploadRequest(BaseModel):
+    """Request model cho upload PDF"""
+    namespace: Optional[str] = Field("Default", description="Namespace trong Pinecone")
+    index_name: Optional[str] = Field("testbot768", description="Tên index trong Pinecone")
+    title: Optional[str] = Field(None, description="Tiêu đề của tài liệu")
+    description: Optional[str] = Field(None, description="Mô tả về tài liệu")
+    vector_database_id: Optional[int] = Field(None, description="ID của vector database trong PostgreSQL để sử dụng")
+
+class PDFResponse(BaseModel):
+    """Response model cho xử lý PDF"""
+    success: bool = Field(..., description="Trạng thái xử lý thành công hay không")
+    document_id: Optional[str] = Field(None, description="ID của tài liệu")
+    chunks_processed: Optional[int] = Field(None, description="Số lượng chunks đã xử lý")
+    total_text_length: Optional[int] = Field(None, description="Tổng độ dài văn bản")
+    error: Optional[str] = Field(None, description="Thông báo lỗi nếu có")
+    
+    class Config:
+        schema_extra = {
+            "example": {
+                "success": True,
+                "document_id": "550e8400-e29b-41d4-a716-446655440000",
+                "chunks_processed": 25,
+                "total_text_length": 50000
+            }
+        }
+
+class DeleteDocumentRequest(BaseModel):
+    """Request model cho xóa document"""
+    document_id: str = Field(..., description="ID của tài liệu cần xóa")
+    namespace: Optional[str] = Field("Default", description="Namespace trong Pinecone")
+    index_name: Optional[str] = Field("testbot768", description="Tên index trong Pinecone")
+
+class DocumentsListResponse(BaseModel):
+    """Response model cho lấy danh sách tài liệu"""
+    success: bool = Field(..., description="Trạng thái xử lý thành công hay không")
+    total_vectors: Optional[int] = Field(None, description="Tổng số vectors trong index")
+    namespace: Optional[str] = Field(None, description="Namespace đang sử dụng")
+    index_name: Optional[str] = Field(None, description="Tên index đang sử dụng")
+    error: Optional[str] = Field(None, description="Thông báo lỗi nếu có")
+    
+    class Config:
+        schema_extra = {
+            "example": {
+                "success": True,
+                "total_vectors": 5000,
+                "namespace": "Default",
+                "index_name": "testbot768"
+            }
+        } 

app/models/rag_models.py

@@ -0,0 +1,68 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict, Any
+
+class ChatRequest(BaseModel):
+    """Request model for chat endpoint"""
+    user_id: str = Field(..., description="User ID from Telegram")
+    question: str = Field(..., description="User's question")
+    include_history: bool = Field(True, description="Whether to include user history in prompt")
+    use_rag: bool = Field(True, description="Whether to use RAG")
+    
+    # Advanced retrieval parameters
+    similarity_top_k: int = Field(6, description="Number of top similar documents to return (after filtering)")
+    limit_k: int = Field(10, description="Maximum number of documents to retrieve from vector store")
+    similarity_metric: str = Field("cosine", description="Similarity metric to use (cosine, dotproduct, euclidean)")
+    similarity_threshold: float = Field(0.75, description="Threshold for vector similarity (0-1)")
+    
+    # User information
+    session_id: Optional[str] = Field(None, description="Session ID for tracking conversations")
+    first_name: Optional[str] = Field(None, description="User's first name")
+    last_name: Optional[str] = Field(None, description="User's last name")
+    username: Optional[str] = Field(None, description="User's username")
+
+class SourceDocument(BaseModel):
+    """Model for source documents"""
+    text: str = Field(..., description="Text content of the document")
+    source: Optional[str] = Field(None, description="Source of the document")
+    score: Optional[float] = Field(None, description="Raw similarity score of the document")
+    normalized_score: Optional[float] = Field(None, description="Normalized similarity score (0-1)")
+    metadata: Optional[Dict[str, Any]] = Field(None, description="Metadata of the document")
+
+class ChatResponse(BaseModel):
+    """Response model for chat endpoint"""
+    answer: str = Field(..., description="Generated answer")
+    processing_time: float = Field(..., description="Processing time in seconds")
+
+class ChatResponseInternal(BaseModel):
+    """Internal model for chat response with sources - used only for logging"""
+    answer: str
+    sources: Optional[List[SourceDocument]] = Field(None, description="Source documents used for generating answer")
+    processing_time: Optional[float] = None
+
+class EmbeddingRequest(BaseModel):
+    """Request model for embedding endpoint"""
+    text: str = Field(..., description="Text to generate embedding for")
+
+class EmbeddingResponse(BaseModel):
+    """Response model for embedding endpoint"""
+    embedding: List[float] = Field(..., description="Generated embedding")
+    text: str = Field(..., description="Text that was embedded")
+    model: str = Field(..., description="Model used for embedding")
+
+class HealthResponse(BaseModel):
+    """Response model for health endpoint"""
+    status: str
+    services: Dict[str, bool]
+    timestamp: str
+
+class UserMessageModel(BaseModel):
+    """Model for user messages sent to the RAG API"""
+    user_id: str = Field(..., description="User ID from the client application")
+    session_id: str = Field(..., description="Session ID for tracking the conversation")
+    message: str = Field(..., description="User's message/question")
+    
+    # Advanced retrieval parameters (optional)
+    similarity_top_k: Optional[int] = Field(None, description="Number of top similar documents to return (after filtering)")
+    limit_k: Optional[int] = Field(None, description="Maximum number of documents to retrieve from vector store")
+    similarity_metric: Optional[str] = Field(None, description="Similarity metric to use (cosine, dotproduct, euclidean)")
+    similarity_threshold: Optional[float] = Field(None, description="Threshold for vector similarity (0-1)") 

app/utils/__init__.py

@@ -0,0 +1 @@
+# Utility functions package 

app/utils/cache.py

@@ -0,0 +1,193 @@
+import os
+import time
+import threading
+import logging
+from typing import Dict, Any, Optional, Tuple, List, Callable, Generic, TypeVar, Union
+from datetime import datetime
+from dotenv import load_dotenv
+import json
+
+# Thiết lập logging
+logger = logging.getLogger(__name__)
+
+# Load biến môi trường
+load_dotenv()
+
+# Cấu hình cache từ biến môi trường
+DEFAULT_CACHE_TTL = int(os.getenv("CACHE_TTL_SECONDS", "300"))  # Mặc định 5 phút
+DEFAULT_CACHE_CLEANUP_INTERVAL = int(os.getenv("CACHE_CLEANUP_INTERVAL", "60"))  # Mặc định 1 phút
+DEFAULT_CACHE_MAX_SIZE = int(os.getenv("CACHE_MAX_SIZE", "1000"))  # Mặc định 1000 phần tử
+
+# Generic type để có thể sử dụng cho nhiều loại giá trị khác nhau
+T = TypeVar('T')
+
+# Cấu trúc cho một phần tử trong cache
+class CacheItem(Generic[T]):
+    def __init__(self, value: T, ttl: int = DEFAULT_CACHE_TTL):
+        self.value = value
+        self.expire_at = time.time() + ttl
+        self.last_accessed = time.time()
+    
+    def is_expired(self) -> bool:
+        """Kiểm tra xem item có hết hạn chưa"""
+        return time.time() > self.expire_at
+    
+    def touch(self) -> None:
+        """Cập nhật thời gian truy cập lần cuối"""
+        self.last_accessed = time.time()
+    
+    def extend(self, ttl: int = DEFAULT_CACHE_TTL) -> None:
+        """Gia hạn thời gian sống của item"""
+        self.expire_at = time.time() + ttl
+
+# Lớp cache chính
+class InMemoryCache:
+    def __init__(
+        self, 
+        ttl: int = DEFAULT_CACHE_TTL,
+        cleanup_interval: int = DEFAULT_CACHE_CLEANUP_INTERVAL,
+        max_size: int = DEFAULT_CACHE_MAX_SIZE
+    ):
+        self.cache: Dict[str, CacheItem] = {}
+        self.ttl = ttl
+        self.cleanup_interval = cleanup_interval
+        self.max_size = max_size
+        self.lock = threading.RLock()  # Sử dụng RLock để tránh deadlock
+        
+        # Khởi động thread dọn dẹp cache định kỳ (active expiration)
+        self.cleanup_thread = threading.Thread(target=self._cleanup_task, daemon=True)
+        self.cleanup_thread.start()
+    
+    def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
+        """Lưu một giá trị vào cache"""
+        with self.lock:
+            ttl_value = ttl if ttl is not None else self.ttl
+            
+            # Nếu cache đã đầy, xóa bớt các item ít được truy cập nhất
+            if len(self.cache) >= self.max_size and key not in self.cache:
+                self._evict_lru_items()
+                
+            self.cache[key] = CacheItem(value, ttl_value)
+            logger.debug(f"Cache set: {key} (expires in {ttl_value}s)")
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Lấy giá trị từ cache. Nếu key không tồn tại hoặc đã hết hạn, trả về giá trị mặc định.
+        Áp dụng lazy expiration: kiểm tra và xóa các item hết hạn khi truy cập.
+        """
+        with self.lock:
+            item = self.cache.get(key)
+            
+            # Nếu không tìm thấy key hoặc item đã hết hạn
+            if item is None or item.is_expired():
+                # Nếu item tồn tại nhưng đã hết hạn, xóa nó (lazy expiration)
+                if item is not None:
+                    logger.debug(f"Cache miss (expired): {key}")
+                    del self.cache[key]
+                else:
+                    logger.debug(f"Cache miss (not found): {key}")
+                return default
+            
+            # Cập nhật thời gian truy cập
+            item.touch()
+            logger.debug(f"Cache hit: {key}")
+            return item.value
+    
+    def delete(self, key: str) -> bool:
+        """Xóa một key khỏi cache"""
+        with self.lock:
+            if key in self.cache:
+                del self.cache[key]
+                logger.debug(f"Cache delete: {key}")
+                return True
+            return False
+    
+    def clear(self) -> None:
+        """Xóa tất cả dữ liệu trong cache"""
+        with self.lock:
+            self.cache.clear()
+            logger.debug("Cache cleared")
+    
+    def get_or_set(self, key: str, callback: Callable[[], T], ttl: Optional[int] = None) -> T:
+        """
+        Lấy giá trị từ cache nếu tồn tại, nếu không thì gọi callback để lấy giá trị
+        và lưu vào cache trước khi trả về.
+        """
+        with self.lock:
+            value = self.get(key)
+            if value is None:
+                value = callback()
+                self.set(key, value, ttl)
+            return value
+    
+    def _cleanup_task(self) -> None:
+        """Thread để dọn dẹp các item đã hết hạn (active expiration)"""
+        while True:
+            time.sleep(self.cleanup_interval)
+            try:
+                self._remove_expired_items()
+            except Exception as e:
+                logger.error(f"Error in cache cleanup task: {e}")
+    
+    def _remove_expired_items(self) -> None:
+        """Xóa tất cả các item đã hết hạn trong cache"""
+        with self.lock:
+            now = time.time()
+            expired_keys = [k for k, v in self.cache.items() if v.is_expired()]
+            for key in expired_keys:
+                del self.cache[key]
+            
+            if expired_keys:
+                logger.debug(f"Cleaned up {len(expired_keys)} expired cache items")
+    
+    def _evict_lru_items(self, count: int = 1) -> None:
+        """Xóa bỏ các item ít được truy cập nhất khi cache đầy"""
+        items = sorted(self.cache.items(), key=lambda x: x[1].last_accessed)
+        for i in range(min(count, len(items))):
+            del self.cache[items[i][0]]
+        logger.debug(f"Evicted {min(count, len(items))} least recently used items from cache")
+    
+    def stats(self) -> Dict[str, Any]:
+        """Trả về thống kê về cache"""
+        with self.lock:
+            now = time.time()
+            total_items = len(self.cache)
+            expired_items = sum(1 for item in self.cache.values() if item.is_expired())
+            memory_usage = self._estimate_memory_usage()
+            return {
+                "total_items": total_items,
+                "expired_items": expired_items,
+                "active_items": total_items - expired_items,
+                "memory_usage_bytes": memory_usage,
+                "memory_usage_mb": memory_usage / (1024 * 1024),
+                "max_size": self.max_size
+            }
+    
+    def _estimate_memory_usage(self) -> int:
+        """Ước tính dung lượng bộ nhớ của cache (gần đúng)"""
+        # Ước tính dựa trên kích thước của các key và giá trị
+        cache_size = sum(len(k) for k in self.cache.keys())
+        for item in self.cache.values():
+            try:
+                # Ước tính kích thước của value (gần đúng)
+                if isinstance(item.value, (str, bytes)):
+                    cache_size += len(item.value)
+                elif isinstance(item.value, (dict, list)):
+                    cache_size += len(json.dumps(item.value))
+                else:
+                    # Giá trị mặc định cho các loại dữ liệu khác
+                    cache_size += 100
+            except:
+                cache_size += 100
+        
+        return cache_size
+
+# Singleton instance
+_cache_instance = None
+
+def get_cache() -> InMemoryCache:
+    """Trả về instance singleton của InMemoryCache"""
+    global _cache_instance
+    if _cache_instance is None:
+        _cache_instance = InMemoryCache()
+    return _cache_instance 

app/utils/debug_utils.py

@@ -0,0 +1,207 @@
+import os
+import sys
+import logging
+import traceback
+import json
+import time
+from datetime import datetime
+import platform
+
+# Try to import psutil, provide fallback if not available
+try:
+    import psutil
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+    logging.warning("psutil module not available. System monitoring features will be limited.")
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+class DebugInfo:
+    """Class containing debug information"""
+    
+    @staticmethod
+    def get_system_info():
+        """Get system information"""
+        try:
+            info = {
+                "os": platform.system(),
+                "os_version": platform.version(),
+                "python_version": platform.python_version(),
+                "cpu_count": os.cpu_count(),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Add information from psutil if available
+            if PSUTIL_AVAILABLE:
+                info.update({
+                    "total_memory": round(psutil.virtual_memory().total / (1024 * 1024 * 1024), 2),  # GB
+                    "available_memory": round(psutil.virtual_memory().available / (1024 * 1024 * 1024), 2),  # GB
+                    "cpu_usage": psutil.cpu_percent(interval=0.1),
+                    "memory_usage": psutil.virtual_memory().percent,
+                    "disk_usage": psutil.disk_usage('/').percent,
+                })
+            else:
+                info.update({
+                    "total_memory": "psutil not available",
+                    "available_memory": "psutil not available",
+                    "cpu_usage": "psutil not available",
+                    "memory_usage": "psutil not available",
+                    "disk_usage": "psutil not available",
+                })
+            
+            return info
+        except Exception as e:
+            logger.error(f"Error getting system info: {e}")
+            return {"error": str(e)}
+    
+    @staticmethod
+    def get_env_info():
+        """Get environment variable information (masking sensitive information)"""
+        try:
+            # List of environment variables to mask values
+            sensitive_vars = [
+                "API_KEY", "SECRET", "PASSWORD", "TOKEN", "AUTH", "MONGODB_URL", 
+                "AIVEN_DB_URL", "PINECONE_API_KEY", "GOOGLE_API_KEY"
+            ]
+            
+            env_vars = {}
+            for key, value in os.environ.items():
+                # Check if environment variable contains sensitive words
+                is_sensitive = any(s in key.upper() for s in sensitive_vars)
+                
+                if is_sensitive and value:
+                    # Mask value displaying only the first 4 characters
+                    masked_value = value[:4] + "****" if len(value) > 4 else "****"
+                    env_vars[key] = masked_value
+                else:
+                    env_vars[key] = value
+                    
+            return env_vars
+        except Exception as e:
+            logger.error(f"Error getting environment info: {e}")
+            return {"error": str(e)}
+    
+    @staticmethod
+    def get_database_status():
+        """Get database connection status"""
+        try:
+            from app.database.postgresql import check_db_connection as check_postgresql
+            from app.database.mongodb import check_db_connection as check_mongodb
+            from app.database.pinecone import check_db_connection as check_pinecone
+            
+            return {
+                "postgresql": check_postgresql(),
+                "mongodb": check_mongodb(),
+                "pinecone": check_pinecone(),
+                "timestamp": datetime.now().isoformat()
+            }
+        except Exception as e:
+            logger.error(f"Error getting database status: {e}")
+            return {"error": str(e)}
+
+class PerformanceMonitor:
+    """Performance monitoring class"""
+    
+    def __init__(self):
+        self.start_time = time.time()
+        self.checkpoints = []
+    
+    def checkpoint(self, name):
+        """Mark a checkpoint and record the time"""
+        current_time = time.time()
+        elapsed = current_time - self.start_time
+        self.checkpoints.append({
+            "name": name,
+            "time": current_time,
+            "elapsed": elapsed
+        })
+        logger.debug(f"Checkpoint '{name}' at {elapsed:.4f}s")
+        return elapsed
+    
+    def get_report(self):
+        """Generate performance report"""
+        if not self.checkpoints:
+            return {"error": "No checkpoints recorded"}
+            
+        total_time = time.time() - self.start_time
+        
+        # Calculate time between checkpoints
+        intervals = []
+        prev_time = self.start_time
+        
+        for checkpoint in self.checkpoints:
+            interval = checkpoint["time"] - prev_time
+            intervals.append({
+                "name": checkpoint["name"],
+                "interval": interval,
+                "elapsed": checkpoint["elapsed"]
+            })
+            prev_time = checkpoint["time"]
+            
+        return {
+            "total_time": total_time,
+            "checkpoint_count": len(self.checkpoints),
+            "intervals": intervals
+        }
+
+class ErrorTracker:
+    """Class to track and record errors"""
+    
+    def __init__(self, max_errors=100):
+        self.errors = []
+        self.max_errors = max_errors
+    
+    def track_error(self, error, context=None):
+        """Record error information"""
+        error_info = {
+            "error_type": type(error).__name__,
+            "error_message": str(error),
+            "traceback": traceback.format_exc(),
+            "timestamp": datetime.now().isoformat(),
+            "context": context or {}
+        }
+        
+        # Add to error list
+        self.errors.append(error_info)
+        
+        # Limit the number of stored errors
+        if len(self.errors) > self.max_errors:
+            self.errors.pop(0)  # Remove oldest error
+            
+        return error_info
+    
+    def get_errors(self, limit=None):
+        """Get list of recorded errors"""
+        if limit is None or limit >= len(self.errors):
+            return self.errors
+        return self.errors[-limit:]  # Return most recent errors
+
+# Initialize global objects
+error_tracker = ErrorTracker()
+performance_monitor = PerformanceMonitor()
+
+def debug_view(request=None):
+    """Create a full debug report"""
+    debug_data = {
+        "system_info": DebugInfo.get_system_info(),
+        "database_status": DebugInfo.get_database_status(),
+        "performance": performance_monitor.get_report(),
+        "recent_errors": error_tracker.get_errors(limit=10),
+        "timestamp": datetime.now().isoformat()
+    }
+    
+    # Add request information if available
+    if request:
+        debug_data["request"] = {
+            "method": request.method,
+            "url": str(request.url),
+            "headers": dict(request.headers),
+            "client": {
+                "host": request.client.host if request.client else "unknown",
+                "port": request.client.port if request.client else "unknown"
+            }
+        }
+    
+    return debug_data 

app/utils/middleware.py

@@ -0,0 +1,109 @@
+from fastapi import Request, status
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+import logging
+import time
+import traceback
+import uuid
+from .utils import get_local_time
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+class RequestLoggingMiddleware(BaseHTTPMiddleware):
+    """Middleware to log requests and responses"""
+    
+    async def dispatch(self, request: Request, call_next):
+        request_id = str(uuid.uuid4())
+        request.state.request_id = request_id
+        
+        # Log request information
+        client_host = request.client.host if request.client else "unknown"
+        logger.info(f"Request [{request_id}]: {request.method} {request.url.path} from {client_host}")
+        
+        # Measure processing time
+        start_time = time.time()
+        
+        try:
+            # Process request
+            response = await call_next(request)
+            
+            # Calculate processing time
+            process_time = time.time() - start_time
+            logger.info(f"Response [{request_id}]: {response.status_code} processed in {process_time:.4f}s")
+            
+            # Add headers
+            response.headers["X-Request-ID"] = request_id
+            response.headers["X-Process-Time"] = str(process_time)
+            
+            return response
+            
+        except Exception as e:
+            # Log error
+            process_time = time.time() - start_time
+            logger.error(f"Error [{request_id}] after {process_time:.4f}s: {str(e)}")
+            logger.error(traceback.format_exc())
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "detail": "Internal server error",
+                    "request_id": request_id,
+                    "timestamp": get_local_time()
+                }
+            )
+
+class ErrorHandlingMiddleware(BaseHTTPMiddleware):
+    """Middleware to handle uncaught exceptions in the application"""
+    
+    async def dispatch(self, request: Request, call_next):
+        try:
+            return await call_next(request)
+        except Exception as e:
+            # Get request_id if available
+            request_id = getattr(request.state, "request_id", str(uuid.uuid4()))
+            
+            # Log error
+            logger.error(f"Uncaught exception [{request_id}]: {str(e)}")
+            logger.error(traceback.format_exc())
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "detail": "Internal server error",
+                    "request_id": request_id,
+                    "timestamp": get_local_time()
+                }
+            )
+
+class DatabaseCheckMiddleware(BaseHTTPMiddleware):
+    """Middleware to check database connections before each request"""
+    
+    async def dispatch(self, request: Request, call_next):
+        # Skip paths that don't need database checks
+        skip_paths = ["/", "/health", "/docs", "/redoc", "/openapi.json"]
+        if request.url.path in skip_paths:
+            return await call_next(request)
+        
+        # Check database connections
+        try:
+            # TODO: Add checks for MongoDB and Pinecone if needed
+            # PostgreSQL check is already done in route handler with get_db() method
+            
+            # Process request normally
+            return await call_next(request)
+            
+        except Exception as e:
+            # Log error
+            logger.error(f"Database connection check failed: {str(e)}")
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                content={
+                    "detail": "Database connection failed",
+                    "timestamp": get_local_time()
+                }
+            ) 

app/utils/pdf_processor.py

@@ -0,0 +1,292 @@
+import os
+import time
+import uuid
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+import logging
+from pinecone import Pinecone
+
+from app.database.pinecone import get_pinecone_index, init_pinecone
+from app.database.postgresql import get_db
+from app.database.models import VectorDatabase
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Initialize embeddings model
+embeddings_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+
+class PDFProcessor:
+    """Class for processing PDF files and creating embeddings"""
+    
+    def __init__(self, index_name="testbot768", namespace="Default", api_key=None, vector_db_id=None, mock_mode=False):
+        """Initialize with Pinecone index name, namespace and API key"""
+        self.index_name = index_name
+        self.namespace = namespace
+        self.pinecone_index = None
+        self.api_key = api_key
+        self.vector_db_id = vector_db_id
+        self.pinecone_client = None
+        self.mock_mode = mock_mode  # Add mock mode for testing
+        
+    def _get_api_key_from_db(self):
+        """Get API key from database if not provided directly"""
+        if self.api_key:
+            return self.api_key
+            
+        if not self.vector_db_id:
+            logger.error("No API key provided and no vector_db_id to fetch from database")
+            return None
+            
+        try:
+            # Get database session
+            db = next(get_db())
+            
+            # Get vector database
+            vector_db = db.query(VectorDatabase).filter(
+                VectorDatabase.id == self.vector_db_id
+            ).first()
+            
+            if not vector_db:
+                logger.error(f"Vector database with ID {self.vector_db_id} not found")
+                return None
+                
+            # Get API key from relationship
+            if hasattr(vector_db, 'api_key_ref') and vector_db.api_key_ref and hasattr(vector_db.api_key_ref, 'key_value'):
+                logger.info(f"Using API key from api_key table for vector database ID {self.vector_db_id}")
+                return vector_db.api_key_ref.key_value
+                
+            logger.error(f"No API key found for vector database ID {self.vector_db_id}. Make sure the api_key_id is properly set.")
+            return None
+        except Exception as e:
+            logger.error(f"Error fetching API key from database: {e}")
+            return None
+    
+    def _init_pinecone_connection(self):
+        """Initialize connection to Pinecone with new API"""
+        try:
+            # If in mock mode, return a mock index
+            if self.mock_mode:
+                logger.info("Running in mock mode - simulating Pinecone connection")
+                class MockPineconeIndex:
+                    def upsert(self, vectors, namespace=None):
+                        logger.info(f"Mock upsert: {len(vectors)} vectors to namespace '{namespace}'")
+                        return {"upserted_count": len(vectors)}
+                        
+                    def delete(self, ids=None, delete_all=False, namespace=None):
+                        logger.info(f"Mock delete: {'all vectors' if delete_all else f'{len(ids)} vectors'} from namespace '{namespace}'")
+                        return {"deleted_count": 10 if delete_all else len(ids or [])}
+                        
+                    def describe_index_stats(self):
+                        logger.info(f"Mock describe_index_stats")
+                        return {"total_vector_count": 100, "namespaces": {self.namespace: {"vector_count": 50}}}
+                
+                return MockPineconeIndex()
+            
+            # Get API key from database if not provided
+            api_key = self._get_api_key_from_db()
+            
+            if not api_key or not self.index_name:
+                logger.error("Pinecone API key or index name not available")
+                return None
+            
+            # Initialize Pinecone client using the new API
+            self.pinecone_client = Pinecone(api_key=api_key)
+            
+            # Get the index
+            index_list = self.pinecone_client.list_indexes()
+            existing_indexes = index_list.names() if hasattr(index_list, 'names') else []
+            
+            if self.index_name not in existing_indexes:
+                logger.error(f"Index {self.index_name} does not exist in Pinecone")
+                return None
+            
+            # Connect to the index
+            index = self.pinecone_client.Index(self.index_name)
+            logger.info(f"Connected to Pinecone index: {self.index_name}")
+            return index
+        except Exception as e:
+            logger.error(f"Error connecting to Pinecone: {e}")
+            return None
+            
+    async def process_pdf(self, file_path, document_id=None, metadata=None, progress_callback=None):
+        """
+        Process PDF file, split into chunks and create embeddings
+        
+        Args:
+            file_path (str): Path to the PDF file
+            document_id (str, optional): Document ID, if not provided a new ID will be created
+            metadata (dict, optional): Additional metadata for the document
+            progress_callback (callable, optional): Callback function for progress updates
+            
+        Returns:
+            dict: Processing result information including document_id and processed chunks count
+        """
+        try:
+            # Initialize Pinecone connection if not already done
+            self.pinecone_index = self._init_pinecone_connection()
+            if not self.pinecone_index:
+                return {"success": False, "error": "Could not connect to Pinecone"}
+            
+            # Create document_id if not provided
+            if not document_id:
+                document_id = str(uuid.uuid4())
+            
+            # Load PDF using PyPDFLoader
+            logger.info(f"Reading PDF file: {file_path}")
+            if progress_callback:
+                await progress_callback("pdf_loading", 0.5, "Loading PDF file")
+                
+            loader = PyPDFLoader(file_path)
+            pages = loader.load()
+            
+            # Extract and concatenate text from all pages
+            all_text = ""
+            for page in pages:
+                all_text += page.page_content + "\n"
+            
+            if progress_callback:
+                await progress_callback("text_extraction", 0.6, "Extracted text from PDF")
+                
+            # Split text into chunks
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=300)
+            chunks = text_splitter.split_text(all_text)
+            
+            logger.info(f"Split PDF file into {len(chunks)} chunks")
+            if progress_callback:
+                await progress_callback("chunking", 0.7, f"Split document into {len(chunks)} chunks")
+            
+            # Process embeddings for each chunk and upsert to Pinecone
+            vectors = []
+            for i, chunk in enumerate(chunks):
+                # Update embedding progress
+                if progress_callback and i % 5 == 0:  # Update every 5 chunks to avoid too many notifications
+                    embedding_progress = 0.7 + (0.3 * (i / len(chunks)))
+                    await progress_callback("embedding", embedding_progress, f"Processing chunk {i+1}/{len(chunks)}")
+                
+                # Create vector embedding for each chunk
+                vector = embeddings_model.embed_query(chunk)
+                
+                # Prepare metadata for vector
+                vector_metadata = {
+                    "document_id": document_id,
+                    "chunk_index": i,
+                    "text": chunk
+                }
+                
+                # Add additional metadata if provided
+                if metadata:
+                    for key, value in metadata.items():
+                        if key not in vector_metadata:
+                            vector_metadata[key] = value
+                
+                # Add vector to list for upserting
+                vectors.append({
+                    "id": f"{document_id}_{i}",
+                    "values": vector,
+                    "metadata": vector_metadata
+                })
+                
+                # Upsert in batches of 100 to avoid overloading
+                if len(vectors) >= 100:
+                    await self._upsert_vectors(vectors)
+                    vectors = []
+            
+            # Upsert any remaining vectors
+            if vectors:
+                await self._upsert_vectors(vectors)
+            
+            logger.info(f"Embedded and saved {len(chunks)} chunks from PDF with document_id: {document_id}")
+            
+            # Final progress update
+            if progress_callback:
+                await progress_callback("completed", 1.0, "PDF processing complete")
+            
+            return {
+                "success": True,
+                "document_id": document_id,
+                "chunks_processed": len(chunks),
+                "total_text_length": len(all_text)
+            }
+            
+        except Exception as e:
+            logger.error(f"Error processing PDF: {str(e)}")
+            if progress_callback:
+                await progress_callback("error", 0, f"Error processing PDF: {str(e)}")
+            return {
+                "success": False,
+                "error": str(e)
+            }
+    
+    async def _upsert_vectors(self, vectors):
+        """Upsert vectors to Pinecone"""
+        try:
+            if not vectors:
+                return
+                
+            # Ensure we have a valid pinecone_index
+            if not self.pinecone_index:
+                self.pinecone_index = self._init_pinecone_connection()
+                if not self.pinecone_index:
+                    raise Exception("Cannot connect to Pinecone")
+            
+            result = self.pinecone_index.upsert(
+                vectors=vectors,
+                namespace=self.namespace
+            )
+            
+            logger.info(f"Upserted {len(vectors)} vectors to Pinecone")
+            return result
+        except Exception as e:
+            logger.error(f"Error upserting vectors: {str(e)}")
+            raise
+    
+    async def delete_namespace(self):
+        """
+        Delete all vectors in the current namespace (equivalent to deleting the namespace).
+        """
+        # Initialize connection if needed
+        self.pinecone_index = self._init_pinecone_connection()
+        if not self.pinecone_index:
+            return {"success": False, "error": "Could not connect to Pinecone"}
+
+        try:
+            # delete_all=True will delete all vectors in the namespace
+            result = self.pinecone_index.delete(
+                delete_all=True,
+                namespace=self.namespace
+            )
+            logger.info(f"Deleted namespace '{self.namespace}' (all vectors).")
+            return {"success": True, "detail": result}
+        except Exception as e:
+            logger.error(f"Error deleting namespace '{self.namespace}': {e}")
+            return {"success": False, "error": str(e)}
+    
+    async def list_documents(self):
+        """Get list of all document_ids from Pinecone"""
+        try:
+            # Initialize Pinecone connection if not already done
+            self.pinecone_index = self._init_pinecone_connection()
+            if not self.pinecone_index:
+                return {"success": False, "error": "Could not connect to Pinecone"}
+            
+            # Get index information
+            stats = self.pinecone_index.describe_index_stats()
+            
+            # Query to get list of all unique document_ids
+            # This method may not be efficient with large datasets, but is the simplest approach
+            # In practice, you should maintain a list of document_ids in a separate database
+            
+            return {
+                "success": True,
+                "total_vectors": stats.get('total_vector_count', 0),
+                "namespace": self.namespace,
+                "index_name": self.index_name
+            }
+        except Exception as e:
+            logger.error(f"Error getting document list: {str(e)}")
+            return {
+                "success": False,
+                "error": str(e)
+            } 

app/utils/utils.py

@@ -0,0 +1,478 @@
+import logging
+import time
+import uuid
+import threading
+import os
+from functools import wraps
+from datetime import datetime, timedelta
+import pytz
+from typing import Callable, Any, Dict, Optional, List, Tuple, Set
+import gc
+import heapq
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+)
+logger = logging.getLogger(__name__)
+
+# Asia/Ho_Chi_Minh timezone
+asia_tz = pytz.timezone('Asia/Ho_Chi_Minh')
+
+def generate_uuid():
+    """Generate a unique identifier"""
+    return str(uuid.uuid4())
+
+def get_current_time():
+    """Get current time in ISO format"""
+    return datetime.now().isoformat()
+
+def get_local_time():
+    """Get current time in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz).strftime("%Y-%m-%d %H:%M:%S")
+
+def get_local_datetime():
+    """Get current datetime object in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz)
+
+# For backward compatibility
+get_vietnam_time = get_local_time
+get_vietnam_datetime = get_local_datetime
+
+def timer_decorator(func: Callable) -> Callable:
+    """
+    Decorator to time function execution and log results.
+    """
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        start_time = time.time()
+        try:
+            result = await func(*args, **kwargs)
+            elapsed_time = time.time() - start_time
+            logger.info(f"Function {func.__name__} executed in {elapsed_time:.4f} seconds")
+            return result
+        except Exception as e:
+            elapsed_time = time.time() - start_time
+            logger.error(f"Function {func.__name__} failed after {elapsed_time:.4f} seconds: {e}")
+            raise
+    return wrapper
+
+def sanitize_input(text):
+    """Sanitize input text"""
+    if not text:
+        return ""
+    # Remove potential dangerous characters or patterns
+    return text.strip()
+
+def truncate_text(text, max_length=100):
+    """
+    Truncate text to given max length and add ellipsis.
+    """
+    if not text or len(text) <= max_length:
+        return text
+    return text[:max_length] + "..."
+
+class CacheStrategy:
+    """Cache loading strategy enumeration"""
+    LAZY = "lazy"  # Only load items into cache when requested
+    EAGER = "eager"  # Preload items into cache at initialization
+    MIXED = "mixed"  # Preload high-priority items, lazy load others
+
+class CacheItem:
+    """Represents an item in the cache with metadata"""
+    def __init__(self, key: str, value: Any, ttl: int = 300, priority: int = 1):
+        self.key = key
+        self.value = value
+        self.expiry = datetime.now() + timedelta(seconds=ttl)
+        self.priority = priority  # Higher number = higher priority
+        self.access_count = 0     # Track number of accesses
+        self.last_accessed = datetime.now()
+        
+    def is_expired(self) -> bool:
+        """Check if the item is expired"""
+        return datetime.now() > self.expiry
+    
+    def touch(self):
+        """Update last accessed time and access count"""
+        self.last_accessed = datetime.now()
+        self.access_count += 1
+        
+    def __lt__(self, other):
+        """For heap comparisons - lower priority items are evicted first"""
+        # First compare priority
+        if self.priority != other.priority:
+            return self.priority < other.priority
+        # Then compare access frequency (less frequently accessed items are evicted first)
+        if self.access_count != other.access_count:
+            return self.access_count < other.access_count
+        # Finally compare last access time (oldest accessed first)
+        return self.last_accessed < other.last_accessed
+
+    def get_size(self) -> int:
+        """Approximate memory size of the cache item in bytes"""
+        try:
+            import sys
+            return sys.getsizeof(self.value) + sys.getsizeof(self.key) + 64  # Additional overhead
+        except:
+            # Default estimate if we can't get the size
+            return 1024
+
+# Enhanced in-memory cache implementation
+class EnhancedCache:
+    def __init__(self, 
+                 strategy: str = "lazy", 
+                 max_items: int = 10000, 
+                 max_size_mb: int = 100,
+                 cleanup_interval: int = 60,
+                 stats_enabled: bool = True):
+        """
+        Initialize enhanced cache with configurable strategy.
+        
+        Args:
+            strategy: Cache loading strategy (lazy, eager, mixed)
+            max_items: Maximum number of items to store in cache
+            max_size_mb: Maximum size of cache in MB
+            cleanup_interval: Interval in seconds to run cleanup
+            stats_enabled: Whether to collect cache statistics
+        """
+        self._cache: Dict[str, CacheItem] = {}
+        self._namespace_cache: Dict[str, Set[str]] = {}  # Tracking keys by namespace
+        self._strategy = strategy
+        self._max_items = max_items
+        self._max_size_bytes = max_size_mb * 1024 * 1024
+        self._current_size_bytes = 0
+        self._stats_enabled = stats_enabled
+        
+        # Statistics
+        self._hits = 0
+        self._misses = 0
+        self._evictions = 0
+        self._total_get_time = 0
+        self._total_set_time = 0
+        
+        # Setup cleanup thread
+        self._last_cleanup = datetime.now()
+        self._cleanup_interval = cleanup_interval
+        self._lock = threading.RLock()
+        
+        if cleanup_interval > 0:
+            self._start_cleanup_thread(cleanup_interval)
+        
+        logger.info(f"Enhanced cache initialized with strategy={strategy}, max_items={max_items}, max_size={max_size_mb}MB")
+    
+    def _start_cleanup_thread(self, interval: int):
+        """Start background thread for periodic cleanup"""
+        def cleanup_worker():
+            while True:
+                time.sleep(interval)
+                try:
+                    self.cleanup()
+                except Exception as e:
+                    logger.error(f"Error in cache cleanup: {e}")
+        
+        thread = threading.Thread(target=cleanup_worker, daemon=True)
+        thread.start()
+        logger.info(f"Cache cleanup thread started with interval {interval}s")
+    
+    def get(self, key: str, namespace: str = None) -> Optional[Any]:
+        """Get value from cache if it exists and hasn't expired"""
+        if self._stats_enabled:
+            start_time = time.time()
+            
+        # Use namespaced key if namespace is provided
+        cache_key = f"{namespace}:{key}" if namespace else key
+        
+        with self._lock:
+            cache_item = self._cache.get(cache_key)
+            
+            if cache_item:
+                if cache_item.is_expired():
+                    # Clean up expired key
+                    self._remove_item(cache_key, namespace)
+                    if self._stats_enabled:
+                        self._misses += 1
+                    value = None
+                else:
+                    # Update access metadata
+                    cache_item.touch()
+                    if self._stats_enabled:
+                        self._hits += 1
+                    value = cache_item.value
+            else:
+                if self._stats_enabled:
+                    self._misses += 1
+                value = None
+                
+            if self._stats_enabled:
+                self._total_get_time += time.time() - start_time
+                
+            return value
+    
+    def set(self, key: str, value: Any, ttl: int = 300, priority: int = 1, namespace: str = None) -> None:
+        """Set a value in the cache with TTL in seconds"""
+        if self._stats_enabled:
+            start_time = time.time()
+            
+        # Use namespaced key if namespace is provided
+        cache_key = f"{namespace}:{key}" if namespace else key
+            
+        with self._lock:
+            # Create cache item
+            cache_item = CacheItem(cache_key, value, ttl, priority)
+            item_size = cache_item.get_size()
+            
+            # Check if we need to make room
+            if (len(self._cache) >= self._max_items or 
+                self._current_size_bytes + item_size > self._max_size_bytes):
+                self._evict_items(item_size)
+            
+            # Update size tracking
+            if cache_key in self._cache:
+                # If replacing, subtract old size first
+                self._current_size_bytes -= self._cache[cache_key].get_size()
+            self._current_size_bytes += item_size
+            
+            # Store the item
+            self._cache[cache_key] = cache_item
+            
+            # Update namespace tracking
+            if namespace:
+                if namespace not in self._namespace_cache:
+                    self._namespace_cache[namespace] = set()
+                self._namespace_cache[namespace].add(cache_key)
+                
+            if self._stats_enabled:
+                self._total_set_time += time.time() - start_time
+    
+    def delete(self, key: str, namespace: str = None) -> None:
+        """Delete a key from the cache"""
+        # Use namespaced key if namespace is provided
+        cache_key = f"{namespace}:{key}" if namespace else key
+        
+        with self._lock:
+            self._remove_item(cache_key, namespace)
+    
+    def _remove_item(self, key: str, namespace: str = None):
+        """Internal method to remove an item and update tracking"""
+        if key in self._cache:
+            # Update size tracking
+            self._current_size_bytes -= self._cache[key].get_size()
+            # Remove from cache
+            del self._cache[key]
+            
+            # Update namespace tracking
+            if namespace and namespace in self._namespace_cache:
+                if key in self._namespace_cache[namespace]:
+                    self._namespace_cache[namespace].remove(key)
+                # Cleanup empty sets
+                if not self._namespace_cache[namespace]:
+                    del self._namespace_cache[namespace]
+    
+    def _evict_items(self, needed_space: int = 0) -> None:
+        """Evict items to make room in the cache"""
+        if not self._cache:
+            return
+            
+        with self._lock:
+            # Convert cache items to a list for sorting
+            items = list(self._cache.values())
+            
+            # Sort by priority, access count, and last accessed time
+            items.sort()  # Uses the __lt__ method of CacheItem
+            
+            # Evict items until we have enough space
+            space_freed = 0
+            evicted_count = 0
+            
+            for item in items:
+                # Stop if we've made enough room
+                if (len(self._cache) - evicted_count <= self._max_items * 0.9 and
+                    (space_freed >= needed_space or 
+                     self._current_size_bytes - space_freed <= self._max_size_bytes * 0.9)):
+                    break
+                    
+                # Skip high priority items unless absolutely necessary
+                if item.priority > 9 and evicted_count < len(items) // 2:
+                    continue
+                    
+                # Evict this item
+                item_size = item.get_size()
+                namespace = item.key.split(':', 1)[0] if ':' in item.key else None
+                self._remove_item(item.key, namespace)
+                
+                space_freed += item_size
+                evicted_count += 1
+                if self._stats_enabled:
+                    self._evictions += 1
+            
+            logger.info(f"Cache eviction: removed {evicted_count} items, freed {space_freed / 1024:.2f}KB")
+    
+    def clear(self, namespace: str = None) -> None:
+        """
+        Clear the cache or a specific namespace
+        """
+        with self._lock:
+            if namespace:
+                # Clear only keys in the specified namespace
+                if namespace in self._namespace_cache:
+                    keys_to_remove = list(self._namespace_cache[namespace])
+                    for key in keys_to_remove:
+                        self._remove_item(key, namespace)
+                    # The namespace should be auto-cleaned in _remove_item
+            else:
+                # Clear the entire cache
+                self._cache.clear()
+                self._namespace_cache.clear()
+                self._current_size_bytes = 0
+                
+            logger.info(f"Cache cleared{' for namespace ' + namespace if namespace else ''}")
+    
+    def cleanup(self) -> None:
+        """Remove expired items and run garbage collection if needed"""
+        with self._lock:
+            now = datetime.now()
+            # Only run if it's been at least cleanup_interval since last cleanup
+            if (now - self._last_cleanup).total_seconds() < self._cleanup_interval:
+                return
+                
+            # Find expired items
+            expired_keys = []
+            for key, item in self._cache.items():
+                if item.is_expired():
+                    expired_keys.append((key, key.split(':', 1)[0] if ':' in key else None))
+            
+            # Remove expired items
+            for key, namespace in expired_keys:
+                self._remove_item(key, namespace)
+            
+            # Update last cleanup time
+            self._last_cleanup = now
+            
+            # Run garbage collection if we removed several items
+            if len(expired_keys) > 100:
+                gc.collect()
+                
+            logger.info(f"Cache cleanup: removed {len(expired_keys)} expired items")
+    
+    def get_stats(self) -> Dict:
+        """Get cache statistics"""
+        with self._lock:
+            if not self._stats_enabled:
+                return {"stats_enabled": False}
+                
+            # Calculate hit rate
+            total_requests = self._hits + self._misses
+            hit_rate = (self._hits / total_requests) * 100 if total_requests > 0 else 0
+            
+            # Calculate average times
+            avg_get_time = (self._total_get_time / total_requests) * 1000 if total_requests > 0 else 0
+            avg_set_time = (self._total_set_time / self._evictions) * 1000 if self._evictions > 0 else 0
+            
+            return {
+                "stats_enabled": True,
+                "item_count": len(self._cache),
+                "max_items": self._max_items,
+                "size_bytes": self._current_size_bytes,
+                "max_size_bytes": self._max_size_bytes,
+                "hits": self._hits,
+                "misses": self._misses,
+                "hit_rate_percent": round(hit_rate, 2),
+                "evictions": self._evictions,
+                "avg_get_time_ms": round(avg_get_time, 3),
+                "avg_set_time_ms": round(avg_set_time, 3),
+                "namespace_count": len(self._namespace_cache),
+                "namespaces": list(self._namespace_cache.keys())
+            }
+    
+    def preload(self, items: List[Tuple[str, Any, int, int]], namespace: str = None) -> None:
+        """
+        Preload a list of items into the cache
+        
+        Args:
+            items: List of (key, value, ttl, priority) tuples
+            namespace: Optional namespace for all items
+        """
+        for key, value, ttl, priority in items:
+            self.set(key, value, ttl, priority, namespace)
+        
+        logger.info(f"Preloaded {len(items)} items into cache{' namespace ' + namespace if namespace else ''}")
+    
+    def get_or_load(self, key: str, loader_func: Callable[[], Any], 
+                   ttl: int = 300, priority: int = 1, namespace: str = None) -> Any:
+        """
+        Get from cache or load using the provided function
+        
+        Args:
+            key: Cache key
+            loader_func: Function to call if cache miss occurs
+            ttl: TTL in seconds
+            priority: Item priority
+            namespace: Optional namespace
+            
+        Returns:
+            Cached or freshly loaded value
+        """
+        # Try to get from cache first
+        value = self.get(key, namespace)
+        
+        # If not in cache, load it
+        if value is None:
+            value = loader_func()
+            # Only cache if we got a valid value
+            if value is not None:
+                self.set(key, value, ttl, priority, namespace)
+                
+        return value
+
+# Load cache configuration from environment variables
+CACHE_STRATEGY = os.getenv("CACHE_STRATEGY", "mixed")
+CACHE_MAX_ITEMS = int(os.getenv("CACHE_MAX_ITEMS", "10000"))
+CACHE_MAX_SIZE_MB = int(os.getenv("CACHE_MAX_SIZE_MB", "100"))
+CACHE_CLEANUP_INTERVAL = int(os.getenv("CACHE_CLEANUP_INTERVAL", "60"))
+CACHE_STATS_ENABLED = os.getenv("CACHE_STATS_ENABLED", "true").lower() in ("true", "1", "yes")
+
+# Initialize the enhanced cache
+cache = EnhancedCache(
+    strategy=CACHE_STRATEGY,
+    max_items=CACHE_MAX_ITEMS,
+    max_size_mb=CACHE_MAX_SIZE_MB,
+    cleanup_interval=CACHE_CLEANUP_INTERVAL,
+    stats_enabled=CACHE_STATS_ENABLED
+)
+
+# Backward compatibility for SimpleCache - for a transition period
+class SimpleCache:
+    def __init__(self):
+        """Legacy SimpleCache implementation that uses EnhancedCache underneath"""
+        logger.warning("SimpleCache is deprecated, please use EnhancedCache directly")
+        
+    def get(self, key: str) -> Optional[Any]:
+        """Get value from cache if it exists and hasn't expired"""
+        return cache.get(key)
+    
+    def set(self, key: str, value: Any, ttl: int = 300) -> None:
+        """Set a value in the cache with TTL in seconds"""
+        cache.set(key, value, ttl)
+    
+    def delete(self, key: str) -> None:
+        """Delete a key from the cache"""
+        cache.delete(key)
+    
+    def clear(self) -> None:
+        """Clear the entire cache"""
+        cache.clear()
+
+def get_host_url(request) -> str:
+    """
+    Get the host URL from a request object.
+    """
+    host = request.headers.get("host", "localhost")
+    scheme = request.headers.get("x-forwarded-proto", "http")
+    return f"{scheme}://{host}"
+
+def format_time(timestamp):
+    """
+    Format a timestamp into a human-readable string.
+    """
+    return timestamp.strftime("%Y-%m-%d %H:%M:%S") 

docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "7860:7860"
+    env_file:
+      - .env
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7860/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    volumes:
+      - ./app:/app/app
+    command: uvicorn app:app --host 0.0.0.0 --port 7860 --reload 

docs/api_documentation.md

@@ -0,0 +1,581 @@
+# API Documentation
+
+## Frontend Setup
+
+```javascript
+// Basic Axios setup
+import axios from 'axios';
+
+const api = axios.create({
+  baseURL: 'https://api.your-domain.com',
+  timeout: 10000,
+  headers: {
+    'Content-Type': 'application/json',
+    'Accept': 'application/json'
+  }
+});
+
+// Error handling
+api.interceptors.response.use(
+  response => response.data,
+  error => {
+    const errorMessage = error.response?.data?.detail || 'An error occurred';
+    console.error('API Error:', errorMessage);
+    return Promise.reject(errorMessage);
+  }
+);
+```
+
+## Caching System
+
+- All GET endpoints support `use_cache=true` parameter (default)
+- Cache TTL: 300 seconds (5 minutes)
+- Cache is automatically invalidated on data changes
+
+## Authentication
+
+Currently no authentication is required. If implemented in the future, use JWT Bearer tokens:
+
+```javascript
+const api = axios.create({
+  // ...other config
+  headers: {
+    // ...other headers
+    'Authorization': `Bearer ${token}`
+  }
+});
+```
+
+## Error Codes
+
+| Code | Description |
+|------|-------------|
+| 400 | Bad Request |
+| 404 | Not Found |
+| 500 | Internal Server Error |
+| 503 | Service Unavailable |
+
+## PostgreSQL Endpoints
+
+### FAQ Endpoints
+
+#### Get FAQs List
+```
+GET /postgres/faq
+```
+
+Parameters:
+- `skip`: Number of items to skip (default: 0)
+- `limit`: Maximum items to return (default: 100)
+- `active_only`: Return only active items (default: false)
+- `use_cache`: Use cached data if available (default: true)
+
+Response:
+```json
+[
+  {
+    "question": "How do I book a hotel?",
+    "answer": "You can book a hotel through our app or website.",
+    "is_active": true,
+    "id": 1,
+    "created_at": "2023-01-01T00:00:00",
+    "updated_at": "2023-01-01T00:00:00"
+  }
+]
+```
+
+Example:
+```javascript
+async function getFAQs() {
+  try {
+    const data = await api.get('/postgres/faq', {
+      params: { active_only: true, limit: 20 }
+    });
+    return data;
+  } catch (error) {
+    console.error('Error fetching FAQs:', error);
+    throw error;
+  }
+}
+```
+
+#### Create FAQ
+```
+POST /postgres/faq
+```
+
+Request Body:
+```json
+{
+  "question": "How do I book a hotel?",
+  "answer": "You can book a hotel through our app or website.",
+  "is_active": true
+}
+```
+
+Response: Created FAQ object
+
+#### Get FAQ Detail
+```
+GET /postgres/faq/{faq_id}
+```
+
+Parameters:
+- `faq_id`: ID of FAQ (required)
+- `use_cache`: Use cached data if available (default: true)
+
+Response: FAQ object
+
+#### Update FAQ
+```
+PUT /postgres/faq/{faq_id}
+```
+
+Parameters:
+- `faq_id`: ID of FAQ to update (required)
+
+Request Body: Partial or complete FAQ object
+Response: Updated FAQ object
+
+#### Delete FAQ
+```
+DELETE /postgres/faq/{faq_id}
+```
+
+Parameters:
+- `faq_id`: ID of FAQ to delete (required)
+
+Response:
+```json
+{
+  "status": "success",
+  "message": "FAQ item 1 deleted"
+}
+```
+
+#### Batch Operations
+
+Create multiple FAQs:
+```
+POST /postgres/faqs/batch
+```
+
+Update status of multiple FAQs:
+```
+PUT /postgres/faqs/batch-update-status
+```
+
+Delete multiple FAQs:
+```
+DELETE /postgres/faqs/batch
+```
+
+### Emergency Contact Endpoints
+
+#### Get Emergency Contacts
+```
+GET /postgres/emergency
+```
+
+Parameters:
+- `skip`: Number of items to skip (default: 0)
+- `limit`: Maximum items to return (default: 100)
+- `active_only`: Return only active items (default: false)
+- `use_cache`: Use cached data if available (default: true)
+
+Response: Array of Emergency Contact objects
+
+#### Create Emergency Contact
+```
+POST /postgres/emergency
+```
+
+Request Body:
+```json
+{
+  "name": "Fire Department",
+  "phone_number": "114",
+  "description": "Fire rescue services",
+  "address": "Da Nang",
+  "location": "16.0544, 108.2022",
+  "priority": 1,
+  "is_active": true
+}
+```
+
+Response: Created Emergency Contact object
+
+#### Get Emergency Contact
+```
+GET /postgres/emergency/{emergency_id}
+```
+
+#### Update Emergency Contact
+```
+PUT /postgres/emergency/{emergency_id}
+```
+
+#### Delete Emergency Contact
+```
+DELETE /postgres/emergency/{emergency_id}
+```
+
+#### Batch Operations
+
+Create multiple Emergency Contacts:
+```
+POST /postgres/emergency/batch
+```
+
+Update status of multiple Emergency Contacts:
+```
+PUT /postgres/emergency/batch-update-status
+```
+
+Delete multiple Emergency Contacts:
+```
+DELETE /postgres/emergency/batch
+```
+
+### Event Endpoints
+
+#### Get Events
+```
+GET /postgres/events
+```
+
+Parameters:
+- `skip`: Number of items to skip (default: 0)
+- `limit`: Maximum items to return (default: 100)
+- `active_only`: Return only active items (default: false)
+- `featured_only`: Return only featured items (default: false)
+- `use_cache`: Use cached data if available (default: true)
+
+Response: Array of Event objects
+
+#### Create Event
+```
+POST /postgres/events
+```
+
+Request Body:
+```json
+{
+  "name": "Da Nang Fireworks Festival",
+  "description": "International Fireworks Festival Da Nang 2023",
+  "address": "Dragon Bridge, Da Nang",
+  "location": "16.0610, 108.2277",
+  "date_start": "2023-06-01T19:00:00",
+  "date_end": "2023-06-01T22:00:00",
+  "price": [
+    {"type": "VIP", "amount": 500000},
+    {"type": "Standard", "amount": 300000}
+  ],
+  "url": "https://danangfireworks.com",
+  "is_active": true,
+  "featured": true
+}
+```
+
+Response: Created Event object
+
+#### Get Event
+```
+GET /postgres/events/{event_id}
+```
+
+#### Update Event
+```
+PUT /postgres/events/{event_id}
+```
+
+#### Delete Event
+```
+DELETE /postgres/events/{event_id}
+```
+
+#### Batch Operations
+
+Create multiple Events:
+```
+POST /postgres/events/batch
+```
+
+Update status of multiple Events:
+```
+PUT /postgres/events/batch-update-status
+```
+
+Delete multiple Events:
+```
+DELETE /postgres/events/batch
+```
+
+### About Pixity Endpoints
+
+#### Get About Pixity
+```
+GET /postgres/about-pixity
+```
+
+Response:
+```json
+{
+  "content": "PiXity is your smart, AI-powered local companion...",
+  "id": 1,
+  "created_at": "2023-01-01T00:00:00",
+  "updated_at": "2023-01-01T00:00:00"
+}
+```
+
+#### Update About Pixity
+```
+PUT /postgres/about-pixity
+```
+
+Request Body:
+```json
+{
+  "content": "PiXity is your smart, AI-powered local companion..."
+}
+```
+
+Response: Updated About Pixity object
+
+### Da Nang Bucket List Endpoints
+
+#### Get Da Nang Bucket List
+```
+GET /postgres/danang-bucket-list
+```
+
+Response: Bucket List object with JSON content string
+
+#### Update Da Nang Bucket List
+```
+PUT /postgres/danang-bucket-list
+```
+
+### Solana Summit Endpoints
+
+#### Get Solana Summit
+```
+GET /postgres/solana-summit
+```
+
+Response: Solana Summit object with JSON content string
+
+#### Update Solana Summit
+```
+PUT /postgres/solana-summit
+```
+
+### Health Check
+```
+GET /postgres/health
+```
+
+Response:
+```json
+{
+  "status": "healthy",
+  "message": "PostgreSQL connection is working",
+  "timestamp": "2023-01-01T00:00:00"
+}
+```
+
+## MongoDB Endpoints
+
+### Session Endpoints
+
+#### Create Session
+```
+POST /session
+```
+
+Request Body:
+```json
+{
+  "user_id": "user123",
+  "query": "How do I book a room?",
+  "timestamp": "2023-01-01T00:00:00",
+  "metadata": {
+    "client_info": "web",
+    "location": "Da Nang"
+  }
+}
+```
+
+Response: Created Session object with session_id
+
+#### Update Session with Response
+```
+PUT /session/{session_id}/response
+```
+
+Request Body:
+```json
+{
+  "response": "You can book a room through our app or website.",
+  "response_timestamp": "2023-01-01T00:00:05",
+  "metadata": {
+    "response_time_ms": 234,
+    "model_version": "gpt-4"
+  }
+}
+```
+
+Response: Updated Session object
+
+#### Get Session
+```
+GET /session/{session_id}
+```
+
+Response: Session object
+
+#### Get User History
+```
+GET /history
+```
+
+Parameters:
+- `user_id`: User ID (required)
+- `limit`: Maximum sessions to return (default: 10)
+- `skip`: Number of sessions to skip (default: 0)
+
+Response:
+```json
+{
+  "user_id": "user123",
+  "sessions": [
+    {
+      "session_id": "60f7a8b9c1d2e3f4a5b6c7d8",
+      "query": "How do I book a room?",
+      "timestamp": "2023-01-01T00:00:00",
+      "response": "You can book a room through our app or website.",
+      "response_timestamp": "2023-01-01T00:00:05"
+    }
+  ],
+  "total_count": 1
+}
+```
+
+#### Health Check
+```
+GET /health
+```
+
+## RAG Endpoints
+
+### Create Embedding
+```
+POST /embedding
+```
+
+Request Body:
+```json
+{
+  "text": "Text to embed"
+}
+```
+
+Response:
+```json
+{
+  "embedding": [0.1, 0.2, 0.3, ...],
+  "dimensions": 1536
+}
+```
+
+### Process Chat Request
+```
+POST /chat
+```
+
+Request Body:
+```json
+{
+  "query": "Can you tell me about Pixity?",
+  "chat_history": [
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hello! How can I help you?"}
+  ]
+}
+```
+
+Response:
+```json
+{
+  "answer": "Pixity is a platform...",
+  "sources": [
+    {
+      "document_id": "doc123",
+      "chunk_id": "chunk456",
+      "chunk_text": "Pixity was founded in...",
+      "relevance_score": 0.92
+    }
+  ]
+}
+```
+
+### Direct RAG Query
+```
+POST /rag
+```
+
+Request Body:
+```json
+{
+  "query": "Can you tell me about Pixity?",
+  "namespace": "about_pixity",
+  "top_k": 3
+}
+```
+
+Response: Query results with relevance scores
+
+### Health Check
+```
+GET /health
+```
+
+## PDF Processing Endpoints
+
+### Upload and Process PDF
+```
+POST /pdf/upload
+```
+
+Form Data:
+- `file`: PDF file (required)
+- `namespace`: Vector database namespace (default: "Default")
+- `index_name`: Vector database index name (default: "testbot768")
+- `title`: Document title (optional)
+- `description`: Document description (optional)
+- `user_id`: User ID for WebSocket updates (optional)
+
+Response: Processing results with document_id
+
+### Delete Documents in Namespace
+```
+DELETE /pdf/namespace
+```
+
+Parameters:
+- `namespace`: Vector database namespace (default: "Default")
+- `index_name`: Vector database index name (default: "testbot768")
+- `user_id`: User ID for WebSocket updates (optional)
+
+Response: Deletion results
+
+### Get Documents List
+```
+GET /pdf/documents
+```
+
+Parameters:
+- `namespace`: Vector database namespace (default: "Default")
+- `index_name`: Vector database index name (default: "testbot768")
+
+Response: List of documents in the namespace 

pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+# Bỏ qua cảnh báo về anyio module và các cảnh báo vận hành nội bộ
+filterwarnings =
+    ignore::pytest.PytestAssertRewriteWarning:.*anyio
+    ignore:.*general_plain_validator_function.* is deprecated.*:DeprecationWarning
+    ignore:.*with_info_plain_validator_function.*:DeprecationWarning
+
+# Cấu hình cơ bản khác
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_* 

requirements.txt

@@ -0,0 +1,47 @@
+# FastAPI
+fastapi==0.103.1
+uvicorn[standard]==0.23.2
+pydantic==2.4.2
+python-dotenv==1.0.0
+websockets==11.0.3
+
+# MongoDB
+pymongo==4.6.1
+dnspython==2.4.2
+
+# PostgreSQL
+sqlalchemy==2.0.20
+pydantic-settings==2.0.3
+psycopg2-binary==2.9.7
+
+# Pinecone & RAG
+pinecone-client==3.0.0
+langchain==0.1.4
+langchain-core==0.1.19
+langchain-community==0.0.14
+langchain-google-genai==0.0.5
+langchain-pinecone==0.0.1
+faiss-cpu==1.7.4
+google-generativeai==0.3.1
+
+# Extras
+pytz==2023.3
+python-multipart==0.0.6
+httpx==0.25.1
+requests==2.31.0
+beautifulsoup4==4.12.2
+redis==5.0.1
+
+# Testing
+prometheus-client==0.17.1
+pytest==7.4.0
+pytest-cov==4.1.0
+watchfiles==0.21.0
+
+# Core dependencies
+starlette==0.27.0
+psutil==5.9.6 
+
+# Upload PDF
+pypdf==3.17.4
+