Update app.py

app.py

@@ -107,6 +107,15 @@ class ModelManager:
 
 model_manager = ModelManager()
 
+# File Handling
+import os
+import json
+import csv
+import xml.etree.ElementTree as ET
+import openpyxl  # for handling .xlsx files
+import pdfplumber
+import docx
+
 # File Handling
 class FileHandler:
     @staticmethod
@@ -118,6 +127,14 @@ class FileHandler:
             return FileHandler._extract_from_docx(file_path)
         elif ext == '.txt':
             return FileHandler._extract_from_txt(file_path)
+        elif ext == '.xml':
+            return FileHandler._extract_from_xml(file_path)
+        elif ext == '.json':
+            return FileHandler._extract_from_json(file_path)
+        elif ext == '.xlsx':
+            return FileHandler._extract_from_xlsx(file_path)
+        elif ext == '.csv':
+            return FileHandler._extract_from_csv(file_path)
         else:
             raise ValueError(f"Unsupported file type: {ext}")
 
@@ -136,6 +153,46 @@ class FileHandler:
         with open(file_path, 'r', encoding='utf-8') as f:
             return f.read()
 
+    @staticmethod
+    def _extract_from_xml(file_path):
+        tree = ET.parse(file_path)
+        root = tree.getroot()
+        return FileHandler._extract_xml_text(root)
+
+    @staticmethod
+    def _extract_xml_text(element):
+        # Recursively extract text from XML elements
+        text = element.text or ""
+        for child in element:
+            text += FileHandler._extract_xml_text(child)
+        return text.strip()
+
+    @staticmethod
+    def _extract_from_json(file_path):
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            return json.dumps(data, indent=4)  # Pretty print JSON for readability
+
+    @staticmethod
+    def _extract_from_xlsx(file_path):
+        workbook = openpyxl.load_workbook(file_path)
+        sheet = workbook.active
+        data = []
+        for row in sheet.iter_rows(values_only=True):
+            data.append('\t'.join([str(cell) for cell in row if cell is not None]))
+        return '\n'.join(data)
+
+    @staticmethod
+    def _extract_from_csv(file_path):
+        with open(file_path, 'r', encoding='utf-8') as f:
+            reader = csv.reader(f)
+            data = []
+            for row in reader:
+                data.append(','.join(row))
+            return '\n'.join(data)
+
+
+
 # Text Processing
 def simple_tokenize(text):
     return text.split()
@@ -210,7 +267,9 @@ def optimize_query(
     #print(llm.invoke('Hello'))
     # Limit max time or set a timeout for LLM to avoid endless execution
     optimized_queries = multi_query_retriever.invoke(query, max_time=30)  # Timeout in seconds
-
+    print(optimized_queries)
+    print('---- optimize query 5 ----')
+    
     return optimized_queries
 
     
@@ -1436,8 +1495,8 @@ def run_automated_tests_and_analyze(*args):
         'model_type': model_types,
         'model_name': [name.strip() for name in model_names.split(',')],
         'split_strategy': split_strategies,
-        'chunk_size': [int(size.strip()) for size in chunk_sizes.split(',')],
-        'overlap_size': [int(size.strip()) for size in overlap_sizes.split(',')],
+        'chunk_size': [int(size.strip()) for size in chunk_sizes.split(',') if size.strip()],
+        'overlap_size': [int(size.strip()) for size in overlap_sizes.split(',') if size.strip()], 
         'vector_store_type': vector_store_types,
         'search_type': search_types,
         'top_k': [int(k.strip()) for k in top_k_values.split(',')],