main.py

from fastapi import FastAPI, UploadFile, File, Form, HTTPException
from fastapi.responses import JSONResponse, StreamingResponse  # Add StreamingResponse
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import Optional, List, Dict
from PIL import Image
import io
import numpy as np
import os
from datetime import datetime
from pymongo import MongoClient
from huggingface_hub import InferenceClient

from embedding_service import JinaClipEmbeddingService
from qdrant_service import QdrantVectorService
from advanced_rag import AdvancedRAG
from cag_service import CAGService
from pdf_parser import PDFIndexer
from multimodal_pdf_parser import MultimodalPDFIndexer
from conversation_service import ConversationService
from tools_service import ToolsService
from intent_classifier import IntentClassifier  # NEW
from scenario_engine import ScenarioEngine  # NEW
from lead_storage_service import LeadStorageService  # NEW
from hybrid_chat_endpoint import hybrid_chat_endpoint  # NEW
from hybrid_chat_stream import hybrid_chat_stream  # NEW: Streaming

# Initialize FastAPI app
app = FastAPI(
    title="Event Social Media Embeddings & ChatbotRAG API",
    description="API để embeddings, search và ChatbotRAG với Jina CLIP v2 + Qdrant + MongoDB + LLM",
    version="2.0.0"
)

# CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Initialize services
print("Initializing services...")
embedding_service = JinaClipEmbeddingService(model_path="jinaai/jina-clip-v2")

collection_name = os.getenv("COLLECTION_NAME", "event_social_media")
qdrant_service = QdrantVectorService(
    collection_name=collection_name,
    vector_size=embedding_service.get_embedding_dimension()
)
print(f"✓ Qdrant collection: {collection_name}")

# MongoDB connection
mongodb_uri = os.getenv("MONGODB_URI", "mongodb+srv://truongtn7122003:7KaI9OT5KTUxWjVI@truongtn7122003.xogin4q.mongodb.net/")
mongo_client = MongoClient(mongodb_uri)
db = mongo_client[os.getenv("MONGODB_DB_NAME", "chatbot_rag")]
documents_collection = db["documents"]
chat_history_collection = db["chat_history"]
print("✓ MongoDB connected")

# Hugging Face token
hf_token = os.getenv("HUGGINGFACE_TOKEN")
if hf_token:
    print("✓ Hugging Face token configured")

# Initialize Advanced RAG (Best Case 2025)
advanced_rag = AdvancedRAG(
    embedding_service=embedding_service,
    qdrant_service=qdrant_service
)
print("✓ Advanced RAG pipeline initialized (with Cross-Encoder)")

# Initialize CAG Service (Semantic Cache)
try:
    cag_service = CAGService(
        embedding_service=embedding_service,
        cache_collection="semantic_cache",
        vector_size=embedding_service.get_embedding_dimension(),
        similarity_threshold=0.9,
        ttl_hours=24
    )
    print("✓ CAG Service initialized (Semantic Caching enabled)")
except Exception as e:
    print(f"Warning: CAG Service initialization failed: {e}")
    print("Continuing without semantic caching...")
    cag_service = None

# Initialize PDF Indexer
pdf_indexer = PDFIndexer(
    embedding_service=embedding_service,
    qdrant_service=qdrant_service,
    documents_collection=documents_collection
)
print("✓ PDF Indexer initialized")

# Initialize Multimodal PDF Indexer
multimodal_pdf_indexer = MultimodalPDFIndexer(
    embedding_service=embedding_service,
    qdrant_service=qdrant_service,
    documents_collection=documents_collection
)
print("✓ Multimodal PDF Indexer initialized")

# Initialize Conversation Service
conversations_collection = db["conversations"]
conversation_service = ConversationService(conversations_collection, max_history=10)
print("✓ Conversation Service initialized")

# Initialize Tools Service
tools_service = ToolsService(base_url="https://www.festavenue.site")
print("✓ Tools Service initialized (Function Calling enabled)")

# Initialize Hybrid Chat Components
intent_classifier = IntentClassifier()
print("✓ Intent Classifier initialized")

scenario_engine = ScenarioEngine(scenarios_dir="scenarios")
print("✓ Scenario Engine initialized")

leads_collection = db["leads"]
lead_storage = LeadStorageService(leads_collection)
print("✓ Lead Storage Service initialized")

print("✓ Services initialized successfully")


# Pydantic models for embeddings
class SearchRequest(BaseModel):
    text: Optional[str] = None
    limit: int = 10
    score_threshold: Optional[float] = None
    text_weight: float = 0.5
    image_weight: float = 0.5


class SearchResponse(BaseModel):
    id: str
    confidence: float
    metadata: dict


class IndexResponse(BaseModel):
    success: bool
    id: str
    message: str


# Pydantic models for ChatbotRAG
class ChatRequest(BaseModel):
    message: str
    session_id: Optional[str] = None  # Multi-turn conversation
    user_id: Optional[str] = None  # User identifier for session tracking
    use_rag: bool = True
    top_k: int = 3
    system_message: Optional[str] = """Bạn là trợ lý AI chuyên biệt cho hệ thống quản lý sự kiện và bán vé. 
Vai trò của bạn là trả lời các câu hỏi CHÍNH XÁC dựa trên dữ liệu được cung cấp từ hệ thống.

Quy tắc tuyệt đối:
- CHỈ trả lời câu hỏi liên quan đến: events, social media posts, PDFs đã upload, và dữ liệu trong knowledge base
- KHÔNG trả lời câu hỏi ngoài phạm vi (tin tức, thời tiết, toán học, lập trình, tư vấn cá nhân, v.v.)
- Nếu câu hỏi nằm ngoài phạm vi: BẮT BUỘC trả lời "Chúng tôi không thể trả lời câu hỏi này vì nó nằm ngoài vùng application xử lí."
- Luôn ưu tiên thông tin từ context được cung cấp"""
    max_tokens: int = 512
    temperature: float = 0.7
    top_p: float = 0.95
    hf_token: Optional[str] = None
    # Advanced RAG options
    use_advanced_rag: bool = True
    use_query_expansion: bool = True
    use_reranking: bool = False  # Disabled - Cross-Encoder not good for Vietnamese
    use_compression: bool = True
    score_threshold: float = 0.5
    # Function calling
    enable_tools: bool = True  # Enable API tool calling


class ChatResponse(BaseModel):
    response: str
    context_used: List[Dict]
    timestamp: str
    rag_stats: Optional[Dict] = None  # Stats from advanced RAG pipeline
    session_id: Optional[str] = None  # Session identifier for multi-turn (auto-generated if not provided)
    tool_calls: Optional[List[Dict]] = None  # Track API calls made


class AddDocumentRequest(BaseModel):
    text: str
    metadata: Optional[Dict] = None


class AddDocumentResponse(BaseModel):
    success: bool
    doc_id: str
    message: str


@app.get("/")
async def root():
    """Health check endpoint with comprehensive API documentation"""
    return {
        "status": "running",
        "service": "ChatbotRAG API",
        "version": "2.0.0",
        "vector_db": "Qdrant",
        "document_db": "MongoDB",        
        "endpoints": {
            "chatbot_rag": {
                "API endpoint": "https://minhvtt-ChatbotRAG.hf.space/",
                "POST /chat": {
                    "description": "Chat với AI sử dụng RAG (Retrieval-Augmented Generation)",
                    "request": {
                        "method": "POST",
                        "content_type": "application/json",
                        "body": {
                            "message": "string (required) - User message/question",
                            "use_rag": "boolean (optional, default: true) - Enable RAG context retrieval",
                            "top_k": "integer (optional, default: 3) - Number of context documents to retrieve",
                            "system_message": "string (optional) - Custom system prompt",
                            "max_tokens": "integer (optional, default: 512) - Max response length",
                            "temperature": "float (optional, default: 0.7, range: 0-1) - Creativity level",
                            "top_p": "float (optional, default: 0.95) - Nucleus sampling",
                            "hf_token": "string (optional) - Hugging Face token (fallback to env)"
                        }
                    },
                    "response": {
                        "response": "string - AI generated response",
                        "context_used": [
                            {
                                "id": "string - Document ID",
                                "confidence": "float - Relevance score",
                                "metadata": {
                                    "text": "string - Retrieved context"
                                }
                            }
                        ],
                        "timestamp": "string - ISO 8601 timestamp"
                    },
                    "example_request": {
                        "message": "Dao có nguy hiểm không?",
                        "use_rag": True,
                        "top_k": 3,
                        "temperature": 0.7
                    },
                    "example_response": {
                        "response": "Dựa trên thông tin trong database, dao được phân loại là vũ khí nguy hiểm. Dao sắc có thể gây thương tích nghiêm trọng nếu không sử dụng đúng cách. Cần tuân thủ các quy định an toàn khi sử dụng.",
                        "context_used": [
                            {
                                "id": "68a3fc14c853d7621e8977b5",
                                "confidence": 0.92,
                                "metadata": {
                                    "text": "Vũ khí"
                                }
                            },
                            {
                                "id": "68a3fc4cc853d7621e8977b6",
                                "confidence": 0.85,
                                "metadata": {
                                    "text": "Con dao sắc"
                                }
                            }
                        ],
                        "timestamp": "2025-10-13T10:30:45.123456"
                    },
                    "notes": [
                        "RAG retrieves relevant context from vector DB before generating response",
                        "LLM uses context to provide accurate, grounded answers",
                        "Requires HUGGINGFACE_TOKEN environment variable or hf_token in request"
                    ]
                },
                "POST /documents": {
                    "description": "Add document to knowledge base for RAG",
                    "request": {
                        "method": "POST",
                        "content_type": "application/json",
                        "body": {
                            "text": "string (required) - Document text content",
                            "metadata": "object (optional) - Additional metadata (source, category, etc.)"
                        }
                    },
                    "response": {
                        "success": "boolean",
                        "doc_id": "string - MongoDB ObjectId",
                        "message": "string - Status message"
                    },
                    "example_request": {
                        "text": "Để tạo event mới: Click nút 'Tạo Event' ở góc trên bên phải màn hình. Điền thông tin sự kiện bao gồm tên, ngày giờ, địa điểm. Click Lưu để hoàn tất.",
                        "metadata": {
                            "source": "user_guide.pdf",
                            "section": "create_event",
                            "page": 5,
                            "category": "tutorial"
                        }
                    },
                    "example_response": {
                        "success": True,
                        "doc_id": "67a9876543210fedcba98765",
                        "message": "Document added successfully with ID: 67a9876543210fedcba98765"
                    }
                },
                "POST /rag/search": {
                    "description": "Search in knowledge base (similar to /search/text but for RAG documents)",
                    "request": {
                        "method": "POST",
                        "content_type": "multipart/form-data",
                        "body": {
                            "query": "string (required) - Search query",
                            "top_k": "integer (optional, default: 5) - Number of results",
                            "score_threshold": "float (optional, default: 0.5) - Minimum relevance score"
                        }
                    },
                    "response": [
                        {
                            "id": "string",
                            "confidence": "float",
                            "metadata": {
                                "text": "string",
                                "source": "string"
                            }
                        }
                    ],
                    "example_request": {
                        "query": "cách tạo sự kiện mới",
                        "top_k": 3,
                        "score_threshold": 0.6
                    }
                },
                "GET /history": {
                    "description": "Get chat conversation history",
                    "request": {
                        "method": "GET",
                        "query_params": {
                            "limit": "integer (optional, default: 10) - Number of messages",
                            "skip": "integer (optional, default: 0) - Pagination offset"
                        }
                    },
                    "response": {
                        "history": [
                            {
                                "user_message": "string",
                                "assistant_response": "string",
                                "context_used": "array",
                                "timestamp": "string - ISO 8601"
                            }
                        ],
                        "total": "integer - Total messages count"
                    },
                    "example_request": "GET /history?limit=5&skip=0",
                    "example_response": {
                        "history": [
                            {
                                "user_message": "Dao có nguy hiểm không?",
                                "assistant_response": "Dao được phân loại là vũ khí...",
                                "context_used": [],
                                "timestamp": "2025-10-13T10:30:45.123456"
                            }
                        ],
                        "total": 15
                    }
                },
                "DELETE /documents/{doc_id}": {
                    "description": "Delete document from knowledge base",
                    "request": {
                        "method": "DELETE",
                        "path_params": {
                            "doc_id": "string - MongoDB ObjectId"
                        }
                    },
                    "response": {
                        "success": "boolean",
                        "message": "string"
                    }
                }
            }
        },
        "usage_examples": {
            "curl_chat": "curl -X POST 'http://localhost:8000/chat' -H 'Content-Type: application/json' -d '{\"message\": \"Dao có nguy hiểm không?\", \"use_rag\": true}'",
            "python_chat": """
import requests

response = requests.post(
    'http://localhost:8000/chat',
    json={
        'message': 'Nút tạo event ở đâu?',
        'use_rag': True,
        'top_k': 3
    }
)
print(response.json()['response'])
            """
        },
        "authentication": {
            "embeddings_apis": "No authentication required",
            "chat_api": "Requires HUGGINGFACE_TOKEN (env variable or request body)"
        },
        "rate_limits": {
            "embeddings": "No limit",
            "chat_with_llm": "Limited by Hugging Face API (free tier: ~1000 requests/hour)"
        },
        "error_codes": {
            "400": "Bad Request - Missing required fields or invalid input",
            "401": "Unauthorized - Invalid Hugging Face token",
            "404": "Not Found - Document ID not found",
            "500": "Internal Server Error - Server or database error"
        },
        "links": {
            "docs": "http://localhost:8000/docs",
            "redoc": "http://localhost:8000/redoc",
            "openapi": "http://localhost:8000/openapi.json"
        }
    }

@app.post("/index", response_model=IndexResponse)
async def index_data(
    id: str = Form(...),
    text: str = Form(...),
    image: Optional[UploadFile] = File(None)
):
    """
    Index data vào vector database

    Body:
    - id: Document ID (event ID, post ID, etc.)
    - text: Text content (tiếng Việt supported)
    - image: Image file (optional)

    Returns:
    - success: True/False
    - id: Document ID
    - message: Status message
    """
    try:
        # Prepare embeddings
        text_embedding = None
        image_embedding = None

        # Encode text (tiếng Việt)
        if text and text.strip():
            text_embedding = embedding_service.encode_text(text)

        # Encode image nếu có
        if image:
            image_bytes = await image.read()
            pil_image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
            image_embedding = embedding_service.encode_image(pil_image)

        # Combine embeddings
        if text_embedding is not None and image_embedding is not None:
            # Average của text và image embeddings
            combined_embedding = np.mean([text_embedding, image_embedding], axis=0)
        elif text_embedding is not None:
            combined_embedding = text_embedding
        elif image_embedding is not None:
            combined_embedding = image_embedding
        else:
            raise HTTPException(status_code=400, detail="Phải cung cấp ít nhất text hoặc image")

        # Normalize
        combined_embedding = combined_embedding / np.linalg.norm(combined_embedding, axis=1, keepdims=True)

        # Index vào Qdrant
        metadata = {
            "text": text,
            "has_image": image is not None,
            "image_filename": image.filename if image else None
        }

        result = qdrant_service.index_data(
            doc_id=id,
            embedding=combined_embedding,
            metadata=metadata
        )

        return IndexResponse(
            success=True,
            id=result["original_id"],  # Trả về MongoDB ObjectId
            message=f"Đã index thành công document {result['original_id']} (Qdrant UUID: {result['qdrant_id']})"
        )

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi index: {str(e)}")


@app.post("/search", response_model=List[SearchResponse])
async def search(
    text: Optional[str] = Form(None),
    image: Optional[UploadFile] = File(None),
    limit: int = Form(10),
    score_threshold: Optional[float] = Form(None),
    text_weight: float = Form(0.5),
    image_weight: float = Form(0.5)
):
    """
    Search similar documents bằng text và/hoặc image

    Body:
    - text: Query text (tiếng Việt supported)
    - image: Query image (optional)
    - limit: Số lượng kết quả (default: 10)
    - score_threshold: Minimum confidence score (0-1)
    - text_weight: Weight cho text search (default: 0.5)
    - image_weight: Weight cho image search (default: 0.5)

    Returns:
    - List of results với id, confidence, và metadata
    """
    try:
        # Prepare query embeddings
        text_embedding = None
        image_embedding = None

        # Encode text query
        if text and text.strip():
            text_embedding = embedding_service.encode_text(text)

        # Encode image query
        if image:
            image_bytes = await image.read()
            pil_image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
            image_embedding = embedding_service.encode_image(pil_image)

        # Validate input
        if text_embedding is None and image_embedding is None:
            raise HTTPException(status_code=400, detail="Phải cung cấp ít nhất text hoặc image để search")

        # Hybrid search với Qdrant
        results = qdrant_service.hybrid_search(
            text_embedding=text_embedding,
            image_embedding=image_embedding,
            text_weight=text_weight,
            image_weight=image_weight,
            limit=limit,
            score_threshold=score_threshold,
            ef=256  # High accuracy search
        )

        # Format response
        return [
            SearchResponse(
                id=result["id"],
                confidence=result["confidence"],
                metadata=result["metadata"]
            )
            for result in results
        ]

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi search: {str(e)}")


@app.post("/search/text", response_model=List[SearchResponse])
async def search_by_text(
    text: str = Form(...),
    limit: int = Form(10),
    score_threshold: Optional[float] = Form(None)
):
    """
    Search chỉ bằng text (tiếng Việt)

    Body:
    - text: Query text (tiếng Việt)
    - limit: Số lượng kết quả
    - score_threshold: Minimum confidence score

    Returns:
    - List of results
    """
    try:
        # Encode text
        text_embedding = embedding_service.encode_text(text)

        # Search
        results = qdrant_service.search(
            query_embedding=text_embedding,
            limit=limit,
            score_threshold=score_threshold,
            ef=256
        )

        return [
            SearchResponse(
                id=result["id"],
                confidence=result["confidence"],
                metadata=result["metadata"]
            )
            for result in results
        ]

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi search: {str(e)}")


@app.post("/search/image", response_model=List[SearchResponse])
async def search_by_image(
    image: UploadFile = File(...),
    limit: int = Form(10),
    score_threshold: Optional[float] = Form(None)
):
    """
    Search chỉ bằng image

    Body:
    - image: Query image
    - limit: Số lượng kết quả
    - score_threshold: Minimum confidence score

    Returns:
    - List of results
    """
    try:
        # Encode image
        image_bytes = await image.read()
        pil_image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
        image_embedding = embedding_service.encode_image(pil_image)

        # Search
        results = qdrant_service.search(
            query_embedding=image_embedding,
            limit=limit,
            score_threshold=score_threshold,
            ef=256
        )

        return [
            SearchResponse(
                id=result["id"],
                confidence=result["confidence"],
                metadata=result["metadata"]
            )
            for result in results
        ]

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi search: {str(e)}")


@app.delete("/delete/{doc_id}")
async def delete_document(doc_id: str):
    """
    Delete document by ID (MongoDB ObjectId hoặc UUID)

    Args:
    - doc_id: Document ID to delete

    Returns:
    - Success message
    """
    try:
        qdrant_service.delete_by_id(doc_id)
        return {"success": True, "message": f"Đã xóa document {doc_id}"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi xóa: {str(e)}")


@app.get("/document/{doc_id}")
async def get_document(doc_id: str):
    """
    Get document by ID (MongoDB ObjectId hoặc UUID)

    Args:
    - doc_id: Document ID (MongoDB ObjectId)

    Returns:
    - Document data
    """
    try:
        doc = qdrant_service.get_by_id(doc_id)
        if doc:
            return {
                "success": True,
                "data": doc
            }
        raise HTTPException(status_code=404, detail=f"Không tìm thấy document {doc_id}")
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi get document: {str(e)}")


@app.get("/stats")
async def get_stats():
    """
    Lấy thông tin thống kê collection

    Returns:
    - Collection statistics
    """
    try:
        info = qdrant_service.get_collection_info()
        return info
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi lấy stats: {str(e)}")


# ============================================
# ChatbotRAG Endpoints
# ============================================

# Import chat endpoint logic
from hybrid_chat_endpoint import hybrid_chat_endpoint

@app.post("/chat", response_model=ChatResponse)
async def chat(request: ChatRequest):
    """
    Hybrid Conversational Chatbot: Scenario FSM + RAG
    
    Features:
    - ✅ Scenario-based flows (giá vé, đặt vé kịch bản)
    - ✅ RAG knowledge retrieval (PDF, documents)
    - ✅ Mid-scenario RAG interruption (answer off-topic questions)
    - ✅ Lead collection (email, phone → MongoDB)
    - ✅ Multi-turn conversations with state management
    - ✅ Function calling (external API integration)
    
    Flow:
    1. User message → Intent classification
    2. Route to: Scenario FSM OR RAG OR Hybrid
    3. Execute flow + save state
    4. Save conversation history
    
    Example 1 - Start Price Inquiry Scenario:
    ```
    POST /chat
    {
      "message": "giá vé bao nhiêu?",
      "use_rag": true
    }
    
    Response:
    {
      "response": "Hello 👋 Bạn muốn xem giá của show nào để mình báo đúng nè?",
      "session_id": "abc-123",
      "mode": "scenario",
      "scenario_active": true
    }
    ```
    
    Example 2 - Continue Scenario:
    ```
    POST /chat
    {
      "message": "Show A",
      "session_id": "abc-123"
    }
    
    Response:
    {
      "response": "Bạn đi 1 mình hay đi nhóm...",
      "mode": "scenario",
      "scenario_active": true
    }
    ```
    
    Example 3 - Mid-scenario RAG Question:
    ```
    POST /chat
    {
      "message": "sự kiện mấy giờ?",
      "session_id": "abc-123"
    }
    # Bot answers from RAG, then resumes scenario
    ```
    
    Example 4 - Pure RAG Query:
    ```
    POST /chat
    {
      "message": "địa điểm sự kiện ở đâu?",
      "use_rag": true
    }
    # Normal RAG response (không trigger scenario)
    ```
    """
    return await hybrid_chat_endpoint(
        request=request,
        conversation_service=conversation_service,
        intent_classifier=intent_classifier,
        embedding_service=embedding_service,  # NEW: Required by handlers
        qdrant_service=qdrant_service,        # NEW: Required by handlers
        tools_service=tools_service,
        advanced_rag=advanced_rag,
        chat_history_collection=chat_history_collection,
        hf_token=hf_token,
        lead_storage=lead_storage
    )


@app.post("/chat/stream")
async def chat_stream(request: ChatRequest):
    """
    Streaming Chat Endpoint (SSE - Server-Sent Events)
    
    Real-time token-by-token response display
    
    Features:
    - ✅ Real-time "typing" effect
    - ✅ Status updates (thinking, searching)
    - ✅ Scenario: Simulated streaming (smooth typing)
    - ✅ RAG: Real LLM streaming
    - ✅ HTTP/2 compatible
    
    Event Types:
    - status: Bot status ("Đang suy nghĩ...", "Đang tìm kiếm...")
    - token: Text chunks
    - metadata: Session ID, context info
    - done: Completion signal
    - error: Error messages
    
    Example - JavaScript Client:
    ```javascript
    const response = await fetch('/chat/stream', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        message: "giá vé bao nhiêu?",
        use_rag: true
      })
    });
    
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    
    while (true) {
      const {done, value} = await reader.read();
      if (done) break;
      
      const chunk = decoder.decode(value);
      const lines = chunk.split('\n\n');
      
      for (const line of lines) {
        if (line.startsWith('event: token')) {
          const data = line.split('data: ')[1];
          displayToken(data); // Append to UI
        }
        else if (line.startsWith('event: done')) {
          console.log('Stream complete');
        }
      }
    }
    ```
    
    Example - EventSource (simpler but less control):
    ```javascript
    // Note: EventSource doesn't support POST, need to use fetch
    const eventSource = new EventSource('/chat/stream?message=hello');
    
    eventSource.addEventListener('token', (e) => {
      displayToken(e.data);
    });
    
    eventSource.addEventListener('done', (e) => {
      eventSource.close();
    });
    ```
    """
    return StreamingResponse(
        hybrid_chat_stream(
            request=request,
            conversation_service=conversation_service,
            intent_classifier=intent_classifier,
            embedding_service=embedding_service,  # For handlers
            qdrant_service=qdrant_service,        # For handlers
            advanced_rag=advanced_rag,
            hf_token=hf_token,
            lead_storage=lead_storage
        ),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "X-Accel-Buffering": "no"  # Disable nginx buffering
        }
    )


@app.get("/chat/history/{session_id}")
async def get_conversation_history(session_id: str, include_metadata: bool = False):
    """
    Get conversation history for a session
    
    Args:
        session_id: Session identifier
        include_metadata: Include metadata (rag_stats, tool_calls) in response
    
    Returns:
        List of messages with role and content
        
    Example:
    ```
    GET /chat/history/abc-123?include_metadata=true
    ```
    """
    if not conversation_service.session_exists(session_id):
        raise HTTPException(
            status_code=404,
            detail=f"Session {session_id} not found or has expired"
        )
    
    history = conversation_service.get_conversation_history(
        session_id,
        include_metadata=include_metadata
    )
    
    session_info = conversation_service.get_session_info(session_id)
    
    return {
        "session_id": session_id,
        "message_count": len(history),
        "messages": history,
        "created_at": session_info.get("created_at") if session_info else None,
        "updated_at": session_info.get("updated_at") if session_info else None
    }


@app.get("/chat/sessions")
async def list_sessions(
    limit: int = 50,
    skip: int = 0,
    sort_by: str = "updated_at",
    user_id: Optional[str] = None  # NEW: Filter by user
):
    """
    List all conversation sessions
    
    Query Parameters:
        limit: Maximum sessions to return (default: 50, max: 100)
        skip: Number of sessions to skip for pagination (default: 0)
        sort_by: Field to sort by - 'created_at' or 'updated_at' (default: updated_at)
        user_id: Filter sessions by user_id (optional)
    
    Returns:
        List of sessions with metadata and message counts
        
    Examples:
    ```
    GET /chat/sessions  # All sessions
    GET /chat/sessions?user_id=user_123  # Only user_123's sessions
    GET /chat/sessions?limit=20&skip=0&sort_by=updated_at
    ```
    """
    # Validate limit
    if limit > 100:
        limit = 100
    if limit < 1:
        limit = 1
    
    # Validate sort_by
    if sort_by not in ["created_at", "updated_at"]:
        raise HTTPException(
            status_code=400,
            detail="sort_by must be 'created_at' or 'updated_at'"
        )
    
    sessions = conversation_service.list_sessions(
        limit=limit,
        skip=skip,
        sort_by=sort_by,
        descending=True,
        user_id=user_id  # NEW: Pass user_id filter
    )
    
    total_sessions = conversation_service.count_sessions(user_id=user_id)  # NEW: Count with filter
    
    return {
        "total": total_sessions,
        "limit": limit,
        "skip": skip,
        "count": len(sessions),
        "user_id": user_id,  # NEW: Include filter in response
        "sessions": sessions
    }


@app.get("/scenarios")
async def list_scenarios():
    """
    Get list of all available scenarios for proactive chat
    
    FE use case:
    - Random pick scenario để bắt đầu chat chủ động
    - Hiển thị menu các scenario available
    
    Returns:
        List of scenarios with metadata
        
    Example:
    ```
    GET /scenarios
    
    Response:
    {
      "scenarios": [
        {
          "scenario_id": "price_inquiry",
          "name": "Hỏi giá vé",
          "description": "Tư vấn giá vé và gửi PDF",
          "triggers": ["giá vé", "bao nhiêu"],
          "category": "sales"
        },
        ...
      ]
    }
    ```
    """
    scenarios_list = []
    
    for scenario_id, scenario_data in scenario_engine.scenarios.items():
        scenarios_list.append({
            "scenario_id": scenario_id,
            "name": scenario_data.get("name", scenario_id),
            "description": scenario_data.get("description", ""),
            "triggers": scenario_data.get("triggers", []),
            "category": scenario_data.get("category", "general"),
            "priority": scenario_data.get("priority", "normal"),
            "estimated_duration": scenario_data.get("estimated_duration", "unknown")
        })
    
    return {
        "total": len(scenarios_list),
        "scenarios": scenarios_list
    }


@app.post("/scenarios/{scenario_id}/start")
async def start_scenario_proactive(
    scenario_id: str,
    request_body: Optional[Dict] = None
):
    """
    Start a scenario proactively with optional initial data
    
    Use cases:
    1. FE picks random scenario
    2. BE triggers scenario based on user action (after purchase, exit intent, etc.)
    3. Inject context data (event_name, mood, etc.)
    
    Example 1 - Simple start:
    ```
    POST /scenarios/price_inquiry/start
    {}
    
    Response:
    {
      "session_id": "abc-123",
      "message": "Hello 👋 Bạn muốn xem giá..."
    }
    ```
    
    Example 2 - With initial data (post-event feedback):
    ```
    POST /scenarios/post_event_feedback/start
    {
      "initial_data": {
        "event_name": "Hòa Nhạc Mùa Xuân",
        "event_date": "2024-11-29",
        "event_id": "evt_123"
      },
      "session_id": "existing-session",  // optional
      "user_id": "user_456"               // optional
    }
    
    Response:
    {
      "session_id": "abc-123",
      "message": "Cảm ơn bạn đã tham dự *Hòa Nhạc Mùa Xuân* hôm qua!"
    }
    ```
    
    Example 3 - Mood recommendation:
    ```
    POST /scenarios/mood_recommendation/start
    {
      "initial_data": {
        "mood": "chill",
        "preferred_genre": "acoustic"
      }
    }
    ```
    """
    # Parse request body
    body = request_body or {}
    initial_data = body.get("initial_data", {})
    session_id = body.get("session_id")
    user_id = body.get("user_id")
    
    # Create or use existing session
    if not session_id:
        session_id = conversation_service.create_session(
            metadata={"started_by": "proactive", "scenario": scenario_id},
            user_id=user_id
        )
    
    # Start scenario with initial data
    result = scenario_engine.start_scenario(scenario_id, initial_data)
    
    if result.get("new_state"):
        conversation_service.set_scenario_state(session_id, result["new_state"])
    
    # Save bot message to history
    conversation_service.add_message(
        session_id,
        "assistant",
        result["message"],
        metadata={"proactive": True, "scenario": scenario_id, "initial_data": initial_data}
    )
    
    return {
        "session_id": session_id,
        "scenario_id": scenario_id,
        "message": result["message"],
        "scenario_active": True,
        "proactive": True
    }


@app.post("/chat/clear-session")
async def clear_chat_session(session_id: str):
    """
    Clear conversation history for a session
    
    Args:
        session_id: Session identifier to clear
    
    Returns:
        Success message
        
    Example:
    ```
    POST /chat/clear-session?session_id=abc-123
    ```
    """
    success = conversation_service.clear_session(session_id)
    
    if success:
        return {
            "success": True,
            "message": f"Session {session_id} cleared successfully"
        }
    else:
        raise HTTPException(
            status_code=404,
            detail=f"Session {session_id} not found or already cleared"
        )


@app.get("/chat/session/{session_id}")
async def get_session_info(session_id: str):
    """
    Get metadata about a conversation session
    
    Args:
        session_id: Session identifier
    
    Returns:
        Session info including creation time and message count
        
    Example:
    ```
    GET /chat/session/abc-123
    ```
    """
    session = conversation_service.get_session_info(session_id)
    
    if not session:
        raise HTTPException(
            status_code=404,
            detail=f"Session {session_id} not found"
        )
    
    # Get message count
    history = conversation_service.get_conversation_history(
        session_id,
        include_metadata=True
    )
    
    return {
        "session_id": session["session_id"],
        "created_at": session["created_at"],
        "updated_at": session["updated_at"],
        "message_count": len(history),
        "metadata": session.get("metadata", {})
    }


@app.post("/documents", response_model=AddDocumentResponse)
async def add_document(request: AddDocumentRequest):
    """
    Add document to knowledge base

    Body:
    - text: Document text
    - metadata: Additional metadata (optional)

    Returns:
    - success: True/False
    - doc_id: MongoDB document ID
    - message: Status message
    """
    try:
        # Save to MongoDB
        doc_data = {
            "text": request.text,
            "metadata": request.metadata or {},
            "created_at": datetime.utcnow()
        }
        result = documents_collection.insert_one(doc_data)
        doc_id = str(result.inserted_id)

        # Generate embedding
        embedding = embedding_service.encode_text(request.text)

        # Index to Qdrant
        qdrant_service.index_data(
            doc_id=doc_id,
            embedding=embedding,
            metadata={
                "text": request.text,
                "source": "api",
                **(request.metadata or {})
            }
        )

        return AddDocumentResponse(
            success=True,
            doc_id=doc_id,
            message=f"Document added successfully with ID: {doc_id}"
        )

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")


@app.post("/documents/upload/pdf")
async def upload_pdf(
    file: UploadFile = File(...),
    metadata: Optional[str] = Form(None)
):
    """
    Upload PDF file and index into knowledge base
    
    Features:
    - Extracts text from PDF
    - Detects image URLs in text/markdown
    - Chunks content intelligently  
    - Indexes all chunks into Qdrant for RAG
    
    Args:
        file: PDF file to upload
        metadata: Optional JSON string with metadata (title, author, etc.)
    
    Returns:
        Success status, document ID, and indexing stats
        
    Example:
    ```bash
    curl -X POST http://localhost:8000/documents/upload/pdf \
      -F "file=@document.pdf" \
      -F 'metadata={"title": "User Guide", "category": "documentation"}'
    ```
    """
    try:
        # Validate file type
        if not file.filename.endswith('.pdf'):
            raise HTTPException(
                status_code=400,
                detail="Only PDF files are supported"
            )
        
        # Read file bytes
        pdf_bytes = await file.read()
        
        # Parse metadata if provided
        import json
        doc_metadata = {}
        if metadata:
            try:
                doc_metadata = json.loads(metadata)
            except json.JSONDecodeError:
                raise HTTPException(
                    status_code=400,
                    detail="Invalid metadata JSON format"
                )
        
        # Generate unique document ID
        from bson import ObjectId
        document_id = str(ObjectId())
        
        # Add upload timestamp
        doc_metadata['uploaded_at'] = datetime.utcnow().isoformat()
        doc_metadata['original_filename'] = file.filename
        
        # Index PDF using multimodal parser
        result = multimodal_pdf_indexer.index_pdf_bytes(
            pdf_bytes=pdf_bytes,
            document_id=document_id,
            filename=file.filename,
            document_metadata=doc_metadata
        )
        
        return {
            "success": True,
            "document_id": document_id,
            "filename": file.filename,
            "chunks_indexed": result['chunks_indexed'],
            "images_found": result.get('images_found', 0),
            "message": f"PDF uploaded and indexed: {result['chunks_indexed']} chunks, {result.get('images_found', 0)} image URLs found"
        }
        
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(
            status_code=500,
            detail=f"Error processing PDF: {str(e)}"
        )


@app.post("/rag/search", response_model=List[SearchResponse])
async def rag_search(
    query: str = Form(...),
    top_k: int = Form(5),
    score_threshold: Optional[float] = Form(0.5)
):
    """
    Search in knowledge base

    Body:
    - query: Search query
    - top_k: Number of results (default: 5)
    - score_threshold: Minimum score (default: 0.5)

    Returns:
    - results: List of matching documents
    """
    try:
        # Generate query embedding
        query_embedding = embedding_service.encode_text(query)

        # Search in Qdrant
        results = qdrant_service.search(
            query_embedding=query_embedding,
            limit=top_k,
            score_threshold=score_threshold
        )

        return [
            SearchResponse(
                id=result["id"],
                confidence=result["confidence"],
                metadata=result["metadata"]
            )
            for result in results
        ]

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")


@app.get("/history")
async def get_history(limit: int = 10, skip: int = 0):
    """
    Get chat history

    Query params:
    - limit: Number of messages to return (default: 10)
    - skip: Number of messages to skip (default: 0)

    Returns:
    - history: List of chat messages
    """
    try:
        history = list(
            chat_history_collection
            .find({}, {"_id": 0})
            .sort("timestamp", -1)
            .skip(skip)
            .limit(limit)
        )

        # Convert datetime to string
        for msg in history:
            if "timestamp" in msg:
                msg["timestamp"] = msg["timestamp"].isoformat()

        return {
            "history": history,
            "total": chat_history_collection.count_documents({})
        }

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")


@app.delete("/documents/{doc_id}")
async def delete_document_from_kb(doc_id: str):
    """
    Delete document from knowledge base

    Args:
    - doc_id: Document ID (MongoDB ObjectId)

    Returns:
    - success: True/False
    - message: Status message
    """
    try:
        # Delete from MongoDB
        result = documents_collection.delete_one({"_id": doc_id})

        # Delete from Qdrant
        if result.deleted_count > 0:
            qdrant_service.delete_by_id(doc_id)
            return {"success": True, "message": f"Document {doc_id} deleted from knowledge base"}
        else:
            raise HTTPException(status_code=404, detail=f"Document {doc_id} not found")

    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")


if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        app,
        host="0.0.0.0",
        port=8000,
        log_level="info"
    )