Sync from GitHub Actions

Dockerfile

@@ -10,7 +10,6 @@ RUN pip install -r requirements.txt
 
 RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
 RUN mkdir -p /app/api/instance && chmod -R 777 /app/api/instance
-RUN python -m nltk.downloader punkt stopwords wordnet omw-1.4
 
 RUN mkdir -p /app/.cache
 ENV HF_HOME=/app/.cache

Lawverse/retrieval/sparse.py

@@ -1,46 +1,49 @@
-import numpy as np
+from __future__ import annotations
+import re
+import sys
+from typing import List, Tuple
 from rank_bm25 import BM25Okapi
+from langchain_core.documents import Document
 from Lawverse.logger import logging
 from Lawverse.exception import ExceptionHandle
-import sys
-import re
-import nltk
-from nltk.corpus import stopwords
-from nltk.stem import WordNetLemmatizer
-from nltk.tokenize import word_tokenize
 
-def _safe_nltk_download(package: str):
-    try:
-        nltk.data.find(package)
-    except LookupError:
-        nltk.download(package.split('/')[-1], quiet=True)
 
-_safe_nltk_download('tokenizers/punkt')
-_safe_nltk_download('corpora/stopwords')
-_safe_nltk_download('corpora/wordnet')
+STOP_WORDS = {
+    "a", "an", "the", "and", "or", "but", "if", "then", "else", "when", "while",
+    "is", "are", "am", "was", "were", "be", "been", "being",
+    "to", "of", "in", "on", "for", "from", "by", "with", "as", "at", "into",
+    "this", "that", "these", "those", "it", "its", "he", "she", "they", "them",
+    "his", "her", "their", "we", "you", "your", "i", "me", "my",
+    "shall", "may", "under", "section", "subsection",
+}
 
-try:
-    stop_words = set(stopwords.words('english'))
-except LookupError:
-    stop_words = set()
 
-lemmatizer = WordNetLemmatizer()
+def bm25_tokenizer(text: str) -> List[str]:
+    if not text:
+        return []
 
-def bm25_tokenizer(text):
-    text = (text or "").lower()
-    text = re.sub(r'[^\w\s]', ' ', text)
-    try:
-        tokens = word_tokenize(text)
-    except LookupError:
-        tokens = text.split()
-    tokens = [lemmatizer.lemmatize(word) for word in tokens if word and word not in stop_words]
-    return tokens
+    text = text.lower()
+    english_tokens = re.findall(r"[a-zA-Z0-9]+", text)
+    bangla_tokens = re.findall(r"[\u0980-\u09FF]+", text)
+    tokens = english_tokens + bangla_tokens
 
-def build_sparse_index(chunks, k1=1.5, b=0.75):
+    cleaned = [
+        token
+        for token in tokens
+        if len(token) > 1 and token not in STOP_WORDS
+    ]
+    return cleaned
+
+
+def build_sparse_index(chunks: List[Document], k1: float = 1.5, b: float = 0.8) -> BM25Okapi:
     try:
-        tokenized_corpus = [bm25_tokenizer(chunk.page_content) for chunk in chunks]
-        bm25 = BM25Okapi(tokenized_corpus, k1=k1, b=b)
+        logging.info("Building sparse BM25 index...")
+        tokenized_corpus = [
+            bm25_tokenizer(chunk.page_content)
+            for chunk in chunks
+        ]
 
+        bm25 = BM25Okapi(tokenized_corpus, k1=k1, b=b)
         logging.info(f"BM25 sparse index successfully built with {len(chunks)} chunks.")
         return bm25
 
@@ -48,18 +51,32 @@ def build_sparse_index(chunks, k1=1.5, b=0.75):
         logging.error(f"Failed to build BM25 sparse index. Error: {e}")
         raise ExceptionHandle(e, sys)
 
-def bm25_retrieve(bm25, chunks, query, top_k=10):
+
+def bm25_retrieve(
+    bm25: BM25Okapi,
+    query: str,
+    chunks: List[Document],
+    top_k: int = 10,
+) -> List[Tuple[Document, float]]:
     try:
-        tokenized_query = bm25_tokenizer(query)
-        scores = bm25.get_scores(tokenized_query)
+        query_tokens = bm25_tokenizer(query)
+        scores = bm25.get_scores(query_tokens)
+
+        ranked = sorted(
+            enumerate(scores),
+            key=lambda item: item[1],
+            reverse=True,
+        )[:top_k]
 
-        num_docs = len(chunks)
-        actual_top_k = min(top_k, num_docs)
-        top_indices = np.argsort(scores)[-actual_top_k:][::-1]
+        results = []
+        for idx, score in ranked:
+            doc = chunks[idx]
+            doc.metadata = dict(doc.metadata or {})
+            doc.metadata["bm25_score"] = float(score)
+            results.append((doc, float(score)))
 
-        logging.info(f"Successfully retrieved top {len(top_indices)} relevant chunks using BM25.")
-        return [(chunks[i], float(scores[i])) for i in top_indices]
+        return results
 
     except Exception as e:
-        logging.error("BM25 retrieval failed")
+        logging.error(f"BM25 retrieval failed. Error: {e}")
         raise ExceptionHandle(e, sys)

api/app.py

@@ -1,7 +1,8 @@
 from flask import Flask, render_template, request, jsonify, session, stream_with_context, Response
 from dotenv import load_dotenv
 import os
-import secrets
+from threading import Lock
+import logging as py_logging
 from Lawverse.pipeline.rag_pipeline import rag_components
 from Lawverse.pipeline.llm_loader import llm
 from Lawverse.memory.langchain_memory import ChatMemory
@@ -10,25 +11,38 @@ from Lawverse.monitoring.dashboard import monitor_bp
 from Lawverse.agents.graph import create_agentic_chain
 from Lawverse.storage.factory import get_chat_store
 from api.auth import auth_bp, login_required
-
+for logger_name in [
+    "httpcore",
+    "httpx",
+    "hpack",
+    "filelock",
+    "sentence_transformers",
+    "urllib3",
+]:
+    py_logging.getLogger(logger_name).setLevel(py_logging.WARNING)
+    
 load_dotenv()
-
 app = Flask(__name__, template_folder="../templates")
-app.secret_key = os.getenv("SECRET_KEY") or secrets.token_hex(32)
+app.secret_key = os.getenv("SECRET_KEY")
 
 app.register_blueprint(auth_bp)
 app.register_blueprint(monitor_bp)
 
 BASE_COMPONENTS = None
+BASE_COMPONENTS_LOCK = Lock()
 active_chains = {}
 
 def get_base_components():
     global BASE_COMPONENTS
+    
+    if BASE_COMPONENTS is not None:
+        return BASE_COMPONENTS
 
-    if BASE_COMPONENTS is None:
-        logging.info("Loading Lawverse RAG base components...")
-        BASE_COMPONENTS = rag_components()
-        logging.info("Lawverse RAG base components loaded successfully.")
+    with BASE_COMPONENTS_LOCK:
+        if BASE_COMPONENTS is None:
+            logging.info("Loading Lawverse RAG base components...")
+            BASE_COMPONENTS = rag_components()
+            logging.info("Lawverse RAG base components loaded successfully.")
 
     return BASE_COMPONENTS
 
@@ -113,67 +127,58 @@ def rag_response():
 @app.route("/get_chats", methods=["GET"])
 @login_required
 def get_chats():
-    chats = []
-    user_id = session.get("user_id")
-    os.makedirs(MEMORY_DIR, exist_ok=True)
-    for file_path in glob.glob(f"{MEMORY_DIR}/*.json"):
-        try:
-            with open(file_path, "r", encoding="utf-8") as f:
-                data = json.load(f)
-
-            if data.get("user_id") == user_id:
-                chats.append({
-                    "chat_id": data.get("chat_id"),
-                    "last_updated": data.get("last_updated"),
-                    "title": data.get("title", f"Chat-{data.get('chat_id')}")
-                })
-
-        except Exception as e:
-            logging.warning(f"Skipping unreadable memory file {file_path}: {e}")
+    try:
+        user_id = str(session.get("user_id"))
+        chats = get_chat_store().list_chats(user_id)
+        return jsonify(chats), 200
 
-    chats.sort(key=lambda x: x.get("last_updated") or x.get("chat_id"), reverse=True)
-    return jsonify(chats)
+    except Exception as e:
+        logging.error(f"Failed to list chats from cloud store: {e}")
+        return jsonify([]), 200
 
 
 @app.route("/load_chat/<chat_id>", methods=["POST"])
 @login_required
 def load_chat(chat_id):
-    user_id = session.get("user_id")
-    memory_path = os.path.join(MEMORY_DIR, f"user_{user_id}_{chat_id}.json")
-    if not os.path.exists(memory_path):
-        return jsonify({"error": "Chat not found"}), 404
+    try:
+        user_id = str(session.get("user_id"))
+        data = get_chat_store().load_chat(user_id, chat_id)
+        if not data:
+            return jsonify({"error": "Chat not found"}), 404
 
-    _, memory_manager = create_agent_session(chat_id=chat_id)
+        _, memory_manager = create_agent_session(chat_id=chat_id)
 
-    messages_list = memory_manager.memory.chat_memory.messages
-    messages = []
+        messages_list = memory_manager.memory.chat_memory.messages
+        messages = []
 
-    for i in range(0, len(messages_list), 2):
-        user_msg = messages_list[i].content if i < len(messages_list) else None
-        ai_msg = messages_list[i + 1].content if i + 1 < len(messages_list) else ""
-        if user_msg:
-            messages.append({
-                "user": user_msg,
-                "ai": ai_msg
-            })
+        for i in range(0, len(messages_list), 2):
+            user_msg = messages_list[i].content if i < len(messages_list) else None
+            ai_msg = messages_list[i + 1].content if i + 1 < len(messages_list) else ""
 
-    return jsonify({
-        "chat_id": chat_id,
-        "title": memory_manager._get_title(),
-        "messages": messages,
-    })
+            if user_msg:
+                messages.append({
+                    "user": user_msg,
+                    "ai": ai_msg
+                })
+
+        return jsonify({
+            "chat_id": chat_id,
+            "title": memory_manager._get_title(),
+            "messages": messages,
+        }), 200
+
+    except Exception as e:
+        logging.error(f"Failed to load chat from cloud store: {e}")
+        return jsonify({"error": "Internal Server Error"}), 500
 
 
 @app.route("/delete_chat/<chat_id>", methods=["DELETE"])
 @login_required
 def delete_chat(chat_id):
     try:
-        user_id = session.get("user_id")
-        memory_path = os.path.join(MEMORY_DIR, f"user_{user_id}_{chat_id}.json")
+        user_id = str(session.get("user_id"))
 
-        if os.path.exists(memory_path):
-            os.remove(memory_path)
-            logging.info(f"Deleted chat for user {user_id}, chat_id: {chat_id}")
+        deleted = get_chat_store().delete_chat(user_id, chat_id)
 
         was_active = chat_id in active_chains
         if was_active:
@@ -183,12 +188,12 @@ def delete_chat(chat_id):
             session.pop("chat_id", None)
 
         return jsonify({
-            "success": True,
+            "success": deleted,
             "was_active": was_active
         }), 200
 
     except Exception as e:
-        logging.error(f"Error deleting chat {chat_id}: {e}")
+        logging.error(f"Error deleting cloud chat {chat_id}: {e}")
         return jsonify({"error": "Internal Server Error"}), 500
 
 if __name__ == "__main__":