Update app.py

app.py

@@ -132,8 +132,15 @@ def loadKB(fileprovided, urlProvided, uploads_dir, request):
         for file in request.files.getlist('files[]'):
             print("File Received>>>" + file.filename)
             file.save(os.path.join(uploads_dir, secure_filename(file.filename)))
-            loader = PyPDFLoader(os.path.join(uploads_dir, secure_filename(file.filename)))
-            documents.extend(loader.load())
+            #loader = PyPDFLoader(os.path.join(uploads_dir, secure_filename(file.filename)))
+            #documents.extend(loader.load())
+            BASE_MODEL = "LLAMA-TOKENIZER"
+            savedModelPath = "./model/" + BASE_MODEL
+            global tokenizer
+            #tokenizer = LlamaTokenizer.from_pretrained(savedModelPath)
+            tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+            separator = "</Q>"
+            documents.extend(PDFChunkerWithSeparator(os.path.join(uploads_dir, secure_filename(file.filename)),separator))
     else:
         loader = TextLoader('Jio.txt')
         documents.extend(loader.load())