test

Api/app/main.py

@@ -1,13 +1,17 @@
+import logging
+
 import uvicorn
 from fastapi import APIRouter, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
-from app.modules.querySearch.routes.querySearch_route import \
-    router as query_search_routes
-from app.modules.uploadDocument.routes.uploadDocument_route import \
-    router as upload_file_routes
+from app.modules.querySearch.routes.querySearch_route import (
+    router as query_search_routes,
+)
+from app.modules.uploadDocument.routes.uploadDocument_route import (
+    router as upload_file_routes,
+)
 
 app = FastAPI()
 
@@ -41,12 +45,21 @@ async def root():
 # Serve static files from the 'out/_next/static' directory
 app.mount("/_next/static", StaticFiles(directory="app/out/_next/static"), name="static")
 
+
 # Serve the main index.html
 @app.get("/")
 def read_root():
     return FileResponse("app/out/index.html")
 
 
+@app.on_event("startup")
+async def startup_event():
+    logging.info("Application is starting up...")
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    logging.info("Application is shutting down...")
 
 
 app.include_router(app_router)

Api/app/modules/model.py

@@ -0,0 +1,6 @@
+from sentence_transformers import SentenceTransformer
+from transformers import pipeline
+
+
+model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
+qa_pipeline = pipeline("question-answering", model="deepset/roberta-base-squad2")

Api/app/modules/querySearch/features/querySearch_feature.py

@@ -2,9 +2,9 @@ import json
 from typing import List, Tuple
 
 import numpy as np
-from fastapi.responses import JSONResponse
-from sentence_transformers import SentenceTransformer
-from transformers import pipeline
+# from fastapi.responses import JSONResponse
+# from sentence_transformers import SentenceTransformer
+# from transformers import pipeline
 
 from app.db_local_storage.files_db import VECTOR_FILES_DIRECTORY
 from app.db_local_storage.in_memory_db import query_response_storage
@@ -45,7 +45,6 @@ class QuerySearchFeature:
 
         query_response_storage.append(response_query)
 
-
         return {
             "message": response["answer"],
             "context_used": context,
@@ -56,7 +55,7 @@ class QuerySearchFeature:
     async def semantic_search(
         query: str, chunks: List[str], embeddings: np.ndarray, model
     ) -> List[str]:
-        query_embedding = model.encode([query])
+        query_embedding = model.encode([query], clean_up_tokenization_spaces=False)
         similarities = np.dot(embeddings, query_embedding.T).flatten()
         top_indices = np.argsort(-similarities)[:3]
         return [chunks[i] for i in top_indices]

Api/app/modules/querySearch/routes/querySearch_route.py

@@ -1,19 +1,19 @@
 from fastapi import APIRouter, Form, HTTPException
 from fastapi.responses import JSONResponse
-from sentence_transformers import SentenceTransformer
-from transformers import pipeline
+# from sentence_transformers import SentenceTransformer
+# from transformers import pipeline
 
-from app.modules.querySearch.features.querySearch_feature import QuerySearchFeature
+from app.db_local_storage.in_memory_db import query_response_storage
 from app.modules.querySearch.controllers.querySearch_controller import (
     QuerySearchController,
 )
-from app.db_local_storage.in_memory_db import query_response_storage
-
+from app.modules.model import model, qa_pipeline
+from app.modules.querySearch.features.querySearch_feature import QuerySearchFeature
 
 router = APIRouter()
 
-model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
-qa_pipeline = pipeline("question-answering", model="deepset/roberta-base-squad2")
+# model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
+# qa_pipeline = pipeline("question-answering", model="deepset/roberta-base-squad2")
 
 querySearchFeature = QuerySearchFeature(model, qa_pipeline)
 querySearchController = QuerySearchController(querySearchFeature)

Api/app/modules/uploadDocument/features/createEmbeddings_feature.py

@@ -1,27 +1,21 @@
 import json
 from typing import List
 
-from sentence_transformers import SentenceTransformer
-
-from app.db_local_storage.files_db import VECTOR_FILES_DIRECTORY
 from app.db_local_storage.vector_files_db import vector_files_db as EMBEDDING_DATA
+from app.modules.model import model, qa_pipeline
 
 
 class CreateEmbeddingsFeature:
 
     @staticmethod
-    def chunk_text(text: str, chunk_size: int = 512) -> List[str]:
+    async def chunk_text(text: str, chunk_size: int = 512) -> List[str]:
         chunks = [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]
         return chunks
 
     @staticmethod
     async def create_embeddings(text: str, filename: str) -> List:
-        # TODO: Check model
-        model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
-        chunks = CreateEmbeddingsFeature.chunk_text(text)
 
-        # with open(VECTOR_FILES_DIRECTORY, "r") as file:
-        #     EMBEDDING_DATA = json.load(file)
+        chunks = CreateEmbeddingsFeature.chunk_text(text)
 
         id = len(EMBEDDING_DATA) + 1
         docoument_index = f"document_{id}"
@@ -31,7 +25,7 @@ class CreateEmbeddingsFeature:
         }
 
         for i, chunk in enumerate(chunks):
-            embedding = model.encode(chunk).tolist()
+            embedding = model.encode(chunk, clean_up_tokenization_spaces=False).tolist()
             embedding_entry = {
                 "embedding": embedding,
                 "metadata": {
@@ -42,9 +36,4 @@ class CreateEmbeddingsFeature:
             }
             EMBEDDING_DATA[docoument_index]["data"].append(embedding_entry)
 
-        # print(EMBEDDING_DATA)
-
-        # with open(VECTOR_FILES_DIRECTORY, "w") as f:
-        #     json.dump(EMBEDDING_DATA, f)
-
         return

Api/app/modules/uploadDocument/features/extractText_feature.py

@@ -10,27 +10,27 @@ from app.db_local_storage.documents_db import documents_text
 
 class ExtractTextFeature:
 
-    @staticmethod
-    def ensure_directory_exists(folder_path: str) -> None:
-        """Ensure that the directory exists."""
-        if not os.path.exists(folder_path):
-            os.makedirs(folder_path)
-
-    @staticmethod
-    async def saveFile(content, filename, directory: str) -> str:
-        """Save the uploaded file to the specified directory."""
-        file_path = os.path.join(directory, filename)
-        with open(file_path, "w") as file:
-            file.write(content)
-        return file_path
-
-    @staticmethod
-    async def save_text_from_pdf(file: UploadFile, text) -> str:
-
-        ExtractTextFeature.ensure_directory_exists(TEXT_FILES_DIRECTORY)
-        await ExtractTextFeature.saveFile(text, file.filename, TEXT_FILES_DIRECTORY)
-
-        return text
+    # @staticmethod
+    # def ensure_directory_exists(folder_path: str) -> None:
+    #     """Ensure that the directory exists."""
+    #     if not os.path.exists(folder_path):
+    #         os.makedirs(folder_path)
+
+    # @staticmethod
+    # async def saveFile(content, filename, directory: str) -> str:
+    #     """Save the uploaded file to the specified directory."""
+    #     file_path = os.path.join(directory, filename)
+    #     with open(file_path, "w") as file:
+    #         file.write(content)
+    #     return file_path
+
+    # @staticmethod
+    # async def save_text_from_pdf(file: UploadFile, text) -> str:
+
+    #     ExtractTextFeature.ensure_directory_exists(TEXT_FILES_DIRECTORY)
+    #     await ExtractTextFeature.saveFile(text, file.filename, TEXT_FILES_DIRECTORY)
+
+    #     return text
 
     @staticmethod
     async def extract_text_from_pdf(file: UploadFile) -> str:

Api/app/modules/uploadDocument/features/uploadDocument_feature.py

@@ -8,30 +8,30 @@ from app.db_local_storage.documents_db import documents_db
 
 class UploadDocumentFeature:
 
-    @staticmethod
-    def ensure_directory_exists(folder_path: str) -> None:
-        """Ensure that the directory exists."""
-        if not os.path.exists(folder_path):
-            os.makedirs(folder_path)
-
-    @staticmethod
-    async def saveFile(document: UploadFile, directory: str) -> str:
-        """Save the uploaded file to the specified directory."""
-        file_path = os.path.join(directory, document.filename)
-        with open(file_path, "wb") as file:
-            content = await document.read()
-            file.write(content)
-        return file
-
-    @staticmethod
-    async def SaveFileMemory(document: UploadFile) -> str:
-        """Save the uploaded file to memory."""
-        UploadDocumentFeature.ensure_directory_exists(FILES_DIRECTORY)
-        id = len(FILES_NAMES_DATABASE) + 1
-        FILES_NAMES_DATABASE[id] = document.filename
-
-        file = await UploadDocumentFeature.saveFile(document, FILES_DIRECTORY)
-        return file
+    # @staticmethod
+    # def ensure_directory_exists(folder_path: str) -> None:
+    #     """Ensure that the directory exists."""
+    #     if not os.path.exists(folder_path):
+    #         os.makedirs(folder_path)
+
+    # @staticmethod
+    # async def saveFile(document: UploadFile, directory: str) -> str:
+    #     """Save the uploaded file to the specified directory."""
+    #     file_path = os.path.join(directory, document.filename)
+    #     with open(file_path, "wb") as file:
+    #         content = await document.read()
+    #         file.write(content)
+    #     return file
+
+    # @staticmethod
+    # async def SaveFileMemory(document: UploadFile) -> str:
+    #     """Save the uploaded file to memory."""
+    #     UploadDocumentFeature.ensure_directory_exists(FILES_DIRECTORY)
+    #     id = len(FILES_NAMES_DATABASE) + 1
+    #     FILES_NAMES_DATABASE[id] = document.filename
+
+    #     file = await UploadDocumentFeature.saveFile(document, FILES_DIRECTORY)
+    #     return file
 
     @staticmethod
     async def uploadFile(document: UploadFile) -> Dict[str, str]:

App/my-app/src/services/api/api.config.js

@@ -1,4 +1,4 @@
-// const baseUrl = "http://localhost:8000/api";
-const baseUrl = "https://abadesalex-docurag.hf.space/api";
+const baseUrl = "http://localhost:8000/api";
+// const baseUrl = "https://abadesalex-docurag.hf.space/api";
 
 export default baseUrl;