code logic completed for uploaded files with qdrant

Loaders/generic.py

@@ -1,4 +1,6 @@
 #import 
+import os
+import tempfile
 from langchain_community.document_loaders.generic import GenericLoader
 from langchain_community.document_loaders.parsers import LanguageParser
 from langchain_text_splitters import Language
@@ -10,7 +12,8 @@ class GenericLoaders:
         self.main_config = main_config
         self.repo_path = repo_path
         self.rag_path_ext= rag_path_ext
-
+    
+    #when files are present in dir
     def loaders(self):
         # Load
         try :
@@ -29,10 +32,61 @@ class GenericLoaders:
             )
             documents = loader.load()
             print(len(documents))
-
         except Exception as e:
             print(e)
             
         return documents
 
+    
+    # which files are uploading in streamlit
+    def code_loader_and_splitters(self):
+        documents = []
+        docs=[]
+        try : 
+            code_files = self.repo_path 
+            langlist = dict(self.main_config['Langlist'])
+            langlist = eval(langlist["lang_exts"])  # Convert the string to a dictionary
+            if code_files:
+                for code_file in code_files:
+                    with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+                        tmp_file.write(code_file.read())
+
+                    lang_ext = str(code_file.name.split(".")[-1])
+                    lang_ext = str('.'+lang_ext)
+                    tmp_file_path = tmp_file.name+lang_ext
+                    tmp_file_path=tmp_file_path.replace("\\","/")
+                    os.rename(tmp_file.name,tmp_file_path )
+                    loader = GenericLoader.from_filesystem(
+                            tmp_file_path,
+                            glob="**/*",
+                            suffixes=[lang_ext],
+                            exclude=["**/non-utf8-encoding.py"],
+                            parser=LanguageParser(parser_threshold=500),
+                        )
+                    
+                    
+                    selected_lang =""
+                    for lang, ext in langlist.items():
+                        if ext.lower() == lang_ext:
+                            selected_lang = lang
+                            print(f"selected lang: {selected_lang}")  # For debugging
+
+                    if selected_lang =="":
+                        raise ValueError("Issue in evaluating techctack from extension")
+                    documents.extend(loader.load())
+                    
+                    splitter = RecursiveCharacterTextSplitter.from_language(
+                    language=selected_lang, chunk_size=2000, chunk_overlap=200
+                    )
+                    docs.extend(splitter.split_documents(documents))
+
+                    # Clean up temporary file
+                    os.remove(tmp_file_path)
+            
+        except Exception as e:
+            raise ValueError(f"Problem Occured in loader and splitter : {e}")
+            
+        return docs
+
+
         

app.py

@@ -1,3 +1,4 @@
+from typing import Sequence
 from langchain_text_splitters import TextSplitter
 import streamlit as st
 import pandas as pd
@@ -32,13 +33,17 @@ main_config.read(config_file_path)
 #Process Code Repo 
 def process_code_repo(main_config,db_option,repo_path,groq_api_key,selected_model, rag_path_ext,query):
     if db_option == "Qdrant":
-        # LOAD
-        obj_loaders = GenericLoaders(main_config,repo_path, rag_path_ext)
-        documents = obj_loaders.loaders()
+        # # LOAD
+        # obj_loaders = GenericLoaders(main_config,repo_path, rag_path_ext)
+        # documents = obj_loaders.loaders()
 
-        # TRANSFORM
-        obj_code_splitters = CodeSplitters(main_config,repo_path,rag_path_ext)
-        docs = obj_code_splitters.code_splitters(documents)
+        # # TRANSFORM
+        # obj_code_splitters = CodeSplitters(main_config,repo_path,rag_path_ext)
+        # docs = obj_code_splitters.code_splitters(documents)
+        
+        #LOAD and SPLIT
+        obj_loaders_splitter = GenericLoaders(main_config,repo_path, rag_path_ext)
+        docs = obj_loaders_splitter.code_loader_and_splitters()
 
         #EMBEDDINGS and stored into VDB
         obj_qdr = QdrantVDB(main_config,repo_path,groq_api_key,selected_model, rag_path_ext,query)
@@ -133,17 +138,29 @@ db_option = st.sidebar.selectbox("Select Vector DB:", ["Qdrant", "Chroma"])
 st.sidebar.header("RAG Document Format")
 file_format = st.sidebar.selectbox("Select Input Document Format:", ["Code Repo", "PDF", "CSV"])
 
+# Check if the section exists
+if not main_config.has_section('Supported_Extensions'):
+    raise ValueError("Section 'Supported_Extensions' not found in config.ini")
+
+# Extract extensions from the config dictionary
+extensions =  main_config['Supported_Extensions']['extensions']
+supported_extensions = eval(extensions)
+accepted_types = [f"{ext}" for ext in supported_extensions]
+
+# Print the list of extensions
+print(accepted_types)
 
 if file_format == "Code Repo":
-    st.sidebar.header("Select Language")
-    lang_options = ["java", "python", "csharp"]  # Replace with your actual models
-    selected_lang = st.sidebar.selectbox("Select Model:", lang_options)
-    if selected_lang == "java":
-        rag_path_ext = ".java"
-    elif selected_lang == "python":
-        rag_path_ext = ".py"
-    elif selected_lang == "csharp":
-        rag_path_ext = ".cs"
+    # st.sidebar.header("Select Language")
+    # lang_options = ["java", "python", "csharp"]  # Replace with your actual models
+    # selected_lang = st.sidebar.selectbox("Select Model:", lang_options)
+    # if selected_lang == "java":
+    #     rag_path_ext = ".java"
+    # elif selected_lang == "python":
+    #     rag_path_ext = ".py"
+    # elif selected_lang == "csharp":
+    #     rag_path_ext = ".cs"
+    rag_path_ext =  accepted_types
 
 elif file_format == "PDF":
     rag_path_ext = ".pdf"
@@ -152,7 +169,8 @@ elif file_format == "CSV":
     rag_path_ext = ".csv"
 
 
-uploaded_files = st.file_uploader(f"Upload a {rag_path_ext} file", type=rag_path_ext,accept_multiple_files=True,)
+
+uploaded_files = st.file_uploader(f"Upload files", type=rag_path_ext,accept_multiple_files=True,)
 
 
 st.sidebar.header("Groq API Key")

config.ini

@@ -1,2 +1,13 @@
 [Langlist]
-lang_exts = {"java":".java","csharp":".cs","python":".py"}
+lang_exts = {"c": ".c", "cpp": ".cpp", "cobol": ".cbl", "elixir": ".ex",
+             "go": ".go", "haskell": ".hs", "html": ".html", "java": ".java",
+             "js": ".js", "kotlin": ".kt", "lua": ".lua", "latex": ".tex",
+             "markdown": ".md", "perl": ".pl", "php": ".php", "proto": ".proto",
+             "python": ".py", "rst": ".rst", "ruby": ".rb", "rust": ".rs",
+             "scala": ".scala", "sol": ".sol", "swift": ".swift", "csharp": ".cs"}
+
+
+[Supported_Extensions]
+extensions = [".cpp", ".go", ".java", ".kt", ".js", ".ts", ".php", ".proto", ".py", ".rst",
+              ".rb", ".rs", ".scala", ".swift", ".md", ".tex", ".html", ".sol", ".cs",
+              ".cob", ".c", ".lua", ".pl", ".hs", ".ex"]