Update app.py

app.py

@@ -68,8 +68,9 @@ class EmbeddingModel:
         self.model = HuggingFaceEmbeddings(model_name=model_name)
         self.max_tokens = max_tokens
 
-    def embed(self, text):
-        return self.model.embed_documents([text])
+    def embed(self, chunks: List[str]):
+        # Embed the list of chunks
+        return self.model.embed_documents(chunks)
 
 def process_files(model_name, split_strategy, chunk_size=500, overlap_size=50, max_tokens=None):
     # File processing
@@ -78,41 +79,44 @@ def process_files(model_name, split_strategy, chunk_size=500, overlap_size=50, m
         file_path = os.path.join(FILES_DIR, file)
         text += FileHandler.extract_text(file_path)
 
-    # Split text
+    # Split text into chunks
     if split_strategy == 'token':
         splitter = TokenTextSplitter(chunk_size=chunk_size, chunk_overlap=overlap_size)
     else:
         splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=overlap_size)
     
     chunks = splitter.split_text(text)
+    
+    # Embed chunks, not the full text
     model = EmbeddingModel(MODELS[model_name], max_tokens=max_tokens)
-    embeddings = model.embed(text)
+    embeddings = model.embed(chunks)
     
     return embeddings, chunks
 
 def search_embeddings(query, model_name, top_k):
     model = HuggingFaceEmbeddings(model_name=MODELS[model_name])
     embeddings = model.embed_query(query)
-    return embeddings
+
+    # Perform FAISS or other similarity-based search over embeddings
+    # This part requires you to build and search a FAISS index with embeddings
+
+    return embeddings  # You would likely return the top-k results here
 
 def calculate_statistics(embeddings):
     # Return time taken, token count, etc.
     return {"tokens": len(embeddings), "time_taken": time.time()}
 
 import shutil
+
 def upload_file(file, model_name, split_strategy, chunk_size, overlap_size, max_tokens, query, top_k):
     # Ensure default values are set if None is passed
-    if chunk_size is None:
-        chunk_size = 100  # Set a default chunk size
-    if overlap_size is None:
-        overlap_size = 0  # Set a default overlap size
-
-    # Convert chunk_size and overlap_size to integers after checking for None
-    try:
-        chunk_size = int(chunk_size)  # Convert chunk_size to int
-        overlap_size = int(overlap_size)  # Convert overlap_size to int
-    except ValueError:
-        return {"error": "Chunk size and overlap size must be valid integers."}
+    chunk_size = int(chunk_size) if chunk_size else 100
+    overlap_size = int(overlap_size) if overlap_size else 0
+
+    # Save uploaded file
+    file_path = os.path.join(FILES_DIR, file.name)
+    with open(file_path, "wb") as f:
+        shutil.copyfileobj(file, f)  # Copy the uploaded file content to the destination
 
     # Process files and get embeddings
     embeddings, chunks = process_files(model_name, split_strategy, chunk_size, overlap_size, max_tokens)