Update app.py

app.py

@@ -4,39 +4,42 @@ import os
 from gtts import gTTS
 from deep_translator import GoogleTranslator
 import logging
-from llama_index import VectorStoreIndex, Document
+from llama_index import VectorStoreIndex, Document, SimpleDirectoryReader
+from llama_index.node_parser import SimpleNodeParser
 from llama_index.llms import HuggingFaceLLM
-from llama_index import ServiceContext
+from llama_index import ServiceContext, set_global_service_context
 import torch
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s')
 
-# Initialize the LLM
+# Initialize the LLM with a smaller context window
 try:
     llm = HuggingFaceLLM(
-        context_window=1024,
+        context_window=512,  # Reduced from 1024
         max_new_tokens=256,
         generate_kwargs={"temperature": 0.7, "do_sample": False},
         tokenizer_name="gpt2",
         model_name="gpt2",
         device_map="auto",
-        tokenizer_kwargs={"max_length": 1024},
+        tokenizer_kwargs={"max_length": 512},  # Reduced from 1024
         model_kwargs={"torch_dtype": torch.float32},
     )
 except ImportError:
     # Fallback if Accelerate is not available
     llm = HuggingFaceLLM(
-        context_window=1024,
+        context_window=512,  # Reduced from 1024
         max_new_tokens=256,
         generate_kwargs={"temperature": 0.7, "do_sample": False},
         tokenizer_name="gpt2",
         model_name="gpt2",
-        tokenizer_kwargs={"max_length": 1024},
+        tokenizer_kwargs={"max_length": 512},  # Reduced from 1024
         model_kwargs={"torch_dtype": torch.float32},
     )
 
-# Initialize the ServiceContext
-service_context = ServiceContext.from_defaults(llm=llm, embed_model="local")
+# Initialize the ServiceContext with a chunk size
+node_parser = SimpleNodeParser.from_defaults(chunk_size=256)  # Adjust chunk size as needed
+service_context = ServiceContext.from_defaults(llm=llm, embed_model="local", node_parser=node_parser)
+set_global_service_context(service_context)
 
 # Initialize the index
 index = None
@@ -70,13 +73,14 @@ audio_language_dict = {
 def index_text(text: str) -> str:
     global index
     try:
-        document = Document(text=text)
+        documents = [Document(text=text)]
         if index is None:
-            index = VectorStoreIndex.from_documents([document], service_context=service_context)
+            index = VectorStoreIndex.from_documents(documents)
         else:
-            index.insert(document)
+            index.insert(documents[0])
         return "Text indexed successfully."
     except Exception as e:
+        logging.error(f"Error in indexing: {str(e)}")
         return f"Error indexing text: {str(e)}"
 
 def chat_with_context(question: str) -> str:
@@ -85,10 +89,14 @@ def chat_with_context(question: str) -> str:
         return "Please index some text first."
     
     try:
-        query_engine = index.as_query_engine()
+        query_engine = index.as_query_engine(
+            similarity_top_k=2,  # Adjust as needed
+            response_mode="compact"
+        )
         response = query_engine.query(question)
         return str(response)
     except Exception as e:
+        logging.error(f"Error in chat: {str(e)}")
         return f"Error in chat: {str(e)}"
 
 # Translation function