Update app.py

app.py

@@ -23,11 +23,11 @@ groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
 # Initialize the embedding model
 embed_model = HuggingFaceEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
 
-# Initialize a local LLM for indexing purposes
-local_llm = HuggingFaceLLM(model_name="gpt2", tokenizer_name="gpt2", context_window=512, max_new_tokens=256)
+# Initialize a local LLM for indexing purposes with reduced context window
+local_llm = HuggingFaceLLM(model_name="gpt2", tokenizer_name="gpt2", context_window=256, max_new_tokens=128)
 
-# Set up node parser for chunking
-node_parser = SimpleNodeParser.from_defaults(chunk_size=256, chunk_overlap=20)
+# Set up node parser for chunking with smaller chunk size
+node_parser = SimpleNodeParser.from_defaults(chunk_size=128, chunk_overlap=20)
 
 # Initialize the ServiceContext with the local LLM and node parser
 service_context = ServiceContext.from_defaults(llm=local_llm, embed_model=embed_model, node_parser=node_parser)
@@ -61,7 +61,7 @@ audio_language_dict = {
     "Malayalam": {"code": "ml"}
 }
 
-def index_text(text: str) -> str:
+ef index_text(text: str) -> str:
     global index
     try:
         documents = [Document(text=text)]
@@ -81,13 +81,13 @@ def chat_with_context(question: str, model: str) -> str:
     
     try:
         query_engine = index.as_query_engine(
-            similarity_top_k=2,
+            similarity_top_k=1,
             response_mode="compact"
         )
         context = query_engine.query(question).response
         
         # Truncate context if it's too long
-        max_context_length = 2048  # Adjust as needed
+        max_context_length = 1024  # Reduced from 2048
         if len(context) > max_context_length:
             context = context[:max_context_length] + "..."
         
@@ -101,14 +101,13 @@ def chat_with_context(question: str, model: str) -> str:
                 }
             ],
             model=model,
-            max_tokens=500  # Limit the response length
+            max_tokens=256  # Reduced from 500
         )
         return chat_completion.choices[0].message.content
     except Exception as e:
         logging.error(f"Error in chat: {str(e)}")
         return f"Error in chat: {str(e)}"
 
-
 # Translation function
 def translate_text(text, target_lang_code):
     try: