Add api indexed doc

app/api/api_file.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Request, Query, status
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, JSONResponse
 import os
 import logging
 import uuid
@@ -36,16 +36,30 @@ def is_url(path: str) -> bool:
 
 file_router = APIRouter()
 
-# Configure logging to file with date-based filenames
-log_filename = f"document_logs_{datetime.now().strftime('%Y-%m-%d')}.txt"
-file_handler = logging.FileHandler(log_filename)
-formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-file_handler.setFormatter(formatter)
+# # Configure logging to file with date-based filenames
+# log_filename = f"document_logs_{datetime.now().strftime('%Y-%m-%d')}.txt"
+# file_handler = logging.FileHandler(log_filename)
+# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+# file_handler.setFormatter(formatter)
 
-# Create a logger for document processing
-doc_logger = logging.getLogger('document_logger')
-doc_logger.setLevel(logging.INFO)
-doc_logger.addHandler(file_handler)
+# # Create a logger for document processing
+# doc_logger = logging.getLogger('document_logger')
+# doc_logger.setLevel(logging.INFO)
+# doc_logger.addHandler(file_handler)
+
+# Configure logging to a single file for indexed links
+INDEXED_LINKS_LOG = "indexed_links.txt"
+indexed_links_handler = logging.FileHandler(INDEXED_LINKS_LOG)
+indexed_links_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+indexed_links_handler.setFormatter(indexed_links_formatter)
+
+# Create a logger for indexed links
+indexed_links_logger = logging.getLogger('indexed_links_logger')
+indexed_links_logger.setLevel(logging.INFO)
+indexed_links_logger.addHandler(indexed_links_handler)
+
+# Ensure that the general logger does not propagate to avoid duplicate logs
+indexed_links_logger.propagate = False
 
 # Also configure the general logger if not already configured
 logging.basicConfig(level=logging.INFO)
@@ -54,7 +68,53 @@ logger = logging.getLogger(__name__)
 from app.search.rag_pipeline import RAGSystem
 from sentence_transformers import SentenceTransformer
 
+# Path to the indexed links log file
+INDEXED_LINKS_LOG = "indexed_links.txt"
+
+@file_router.get("/indexed_links/", response_class=JSONResponse)
+async def get_indexed_links(
+    limit: Optional[int] = Query(
+        None, 
+        description="Maximum number of indexed links to return. If not specified, all links are returned."
+    )
+) -> dict:
+    """
+    Retrieve a list of all indexed URLs from the single log file.
 
+    Args:
+        limit (int, optional): Maximum number of indexed links to return.
+
+    Returns:
+        dict: A dictionary containing the list of indexed URLs.
+    """
+    try:
+        if not os.path.exists(INDEXED_LINKS_LOG):
+            raise HTTPException(status_code=404, detail="No indexed links found.")
+
+        indexed_links = []
+
+        with open(INDEXED_LINKS_LOG, "r", encoding="utf-8") as log_file:
+            for line in log_file:
+                # Assuming each log entry is in the format: 'timestamp - level - URL'
+                parts = line.strip().split(" - ")
+                if len(parts) >= 3:
+                    url = parts[2]
+                    indexed_links.append(url)
+
+        if limit is not None:
+            if limit < 0:
+                raise HTTPException(status_code=400, detail="Limit must be a positive integer.")
+            indexed_links = indexed_links[:limit]
+
+        return {"indexed_links": indexed_links}
+
+    except HTTPException as he:
+        raise he  # Re-raise HTTP exceptions to be handled by FastAPI
+    except Exception as e:
+        # Log the exception details for debugging
+        logger.exception("Error retrieving indexed links")
+        raise HTTPException(status_code=500, detail="Internal server error while retrieving indexed links.")
+    
 @file_router.post("/load_file_with_markdown/")
 async def load_file_with_markdown(request: Request, filepaths: List[str]):
     try:
@@ -93,7 +153,9 @@ async def load_file_with_markdown(request: Request, filepaths: List[str]):
                     # Log the ID and a 100-character snippet of the document
                     snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                     # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
-                    doc_logger.info(f"ID: {doc_id}_{document.title}, Snippet: {snippet}")
+                    # doc_logger.info(f"ID: {doc_id}_{path}, Snippet: {snippet}")
+                        # Log the indexed URL to the single log file
+                    indexed_links_logger.info(f"{doc_id}_{path}")
 
 
                 except Exception as e:
@@ -181,7 +243,7 @@ async def load_file_with_markdown_function(filepaths: List[str],
                     # Log the ID and a 100-character snippet of the document
                     snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                     # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
-                    doc_logger(f"ID: {doc_id}, Snippet: {snippet}")
+                    # doc_logger(f"ID: {doc_id}, Snippet: {snippet}")
                     logger.info(f"ID: {doc_id}, Snippet: {snippet}")
 
                 except Exception as e:

app/services/message.py

@@ -142,17 +142,18 @@ async def generate_response_from_gemini(
         logger.info(f"Generating response for sender: {sender}")
 
         # Initialize the model
-        model = genai.GenerativeModel("gemini-1.5-pro-002", system_instruction= system_prompt)
-
+        # model = genai.GenerativeModel("gemini-1.5-pro-002", system_instruction= system_prompt)
+        model = genai.GenerativeModel("gemini-1.5-flash", system_instruction= system_prompt)
         # Start chat with history
         chat = model.start_chat(history=history)
 
         if content:
             if rag_system:
                 keywords = extract_keywords_async(content)
+                # keywords = []
                 logger.info(f"Extracted Keywords: {keywords}")
                 # Implement RAG: Retrieve relevant documents
-                retrieved_docs = await rag_system.adv_query(content, keywords=keywords, top_k=1)
+                retrieved_docs = await rag_system.adv_query(content, keywords=keywords, top_k=5)
                 if retrieved_docs:
                     logger.info(f"Retrieved {len(retrieved_docs)} documents for context.")
                     # Format the retrieved documents as a context string

indexed_links.txt

@@ -0,0 +1,5 @@
+2024-12-26 15:02:10,578 - INFO - 06c0cc2a-ec63-4458-a4e9-ea6e96e0bbb4_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
+2024-12-26 15:02:11,146 - INFO - 2c5a8065-7df2-42d0-8e1e-0e59804c6f25_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
+2024-12-26 15:02:11,848 - INFO - 521bfbd9-654a-4cf9-8b6c-79de4b8e5cd2_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
+2024-12-26 15:02:12,549 - INFO - e4088c2a-b9ef-4c04-b2f8-eb703279ba21_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
+2024-12-26 15:02:13,236 - INFO - 458cc97a-93db-44bf-b7f8-e60c22da2a5e_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase