added changes

requirements.txt

@@ -0,0 +1,11 @@
+pypdf
+langchain
+torch
+accelerate
+bitsandbytes
+ctransformers
+sentence_transformers
+faiss_cpu
+chainlit
+huggingface_hub
+langchain_community

upload.py

@@ -0,0 +1,66 @@
+import streamlit as st
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import os
+import tempfile
+
+DB_FAISS_PATH = 'vectorstore/db_faiss'
+
+def create_vector_db(uploaded_files):
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Save uploaded files to temporary directory
+        for file in uploaded_files:
+            if file.name.endswith('.pdf'):
+                temp_path = os.path.join(temp_dir, file.name)
+                with open(temp_path, "wb") as f:
+                    f.write(file.getvalue())
+
+        # Load PDFs
+        documents = []
+        for file in os.listdir(temp_dir):
+            if file.endswith('.pdf'):
+                pdf_path = os.path.join(temp_dir, file)
+                loader = PyPDFLoader(pdf_path)
+                documents.extend(loader.load())
+
+        # Split documents
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=500,
+            chunk_overlap=50
+        )
+        texts = text_splitter.split_documents(documents)
+
+        # Create embeddings
+        embeddings = HuggingFaceEmbeddings(
+            model_name='sentence-transformers/all-MiniLM-L6-v2',
+            model_kwargs={'device': 'cpu'}
+        )
+
+        # Create and save FAISS database
+        db = FAISS.from_documents(texts, embeddings)
+        db.save_local(DB_FAISS_PATH)
+        return True
+
+def main():
+    st.title("PDF to Vector Database Converter")
+    
+    uploaded_files = st.file_uploader(
+        "Upload PDF files",
+        type=['pdf'],
+        accept_multiple_files=True
+    )
+
+    if st.button("Create Vector Database") and uploaded_files:
+        with st.spinner("Creating vector database..."):
+            try:
+                success = create_vector_db(uploaded_files)
+                if success:
+                    st.success("Vector database created successfully!")
+            except Exception as e:
+                st.error(f"An error occurred: {str(e)}")
+
+if __name__ == "__main__":
+    main()

vectorstore/db_faiss/requirements.txt

@@ -0,0 +1,11 @@
+pypdf
+langchain
+torch
+accelerate
+bitsandbytes
+ctransformers
+sentence_transformers
+faiss_cpu
+chainlit
+huggingface_hub
+langchain_community