Update app.py

app.py

@@ -5,76 +5,100 @@ import streamlit as st
 from huggingface_hub import HfApi, login
 from dotenv import load_dotenv
 
+from download_repo import download_gitlab_repo_to_hfspace
+from process_repo import extract_repo_files
+from chunking import chunk_pythoncode_and_add_metadata, chunk_text_and_add_metadata
+from vectorstore import setup_vectorstore
 from llm import get_groq_llm
 from vectorstore import get_chroma_vectorstore
 from embeddings import get_SFR_Code_embedding_model
 from kadi_apy_bot import KadiAPYBot
+from repo_versions import store_message_from_json
 
 # Load environment variables from .env file
 load_dotenv()
 
-vectorstore_path = "data/vectorstore"
+# Load configuration from JSON file
+
+with open("config.json", "r") as file:
+    config = json.load(file)
 
 GROQ_API_KEY = os.environ["GROQ_API_KEY"]
 HF_TOKEN = os.environ["HF_Token"]
 
-with open("config.json", "r") as file:
-    config = json.load(file)
+
+VECTORSTORE_DIRECTORY = config["vectorstore_directory"]
+CHUNK_SIZE = config["chunking"]["chunk_size"]
+CHUNK_OVERLAP = config["chunking"]["chunk_overlap"]
+
+EMBEDDING_MODEL_NAME = config["embedding_model"]["name"]
+EMBEDDING_MODEL_VERSION = config["embedding_model"]["version"]
+
+LLM_MODEL_NAME = config["llm_model"]["name"]
+LLM_MODEL_TEMPERATURE = config["llm_model"]["temperature"]
+
+GITLAB_API_URL = config["gitlab"]["api_url"]
+GITLAB_PROJECT_ID = config["gitlab"]["project id"]
+GITLAB_PROJECT_VERSION = config["gitlab"]["project version"]
+
+DATA_DIR = config["data_dir"]
+HF_SPACE_NAME = config["hf_space_name"]
 
 login(HF_TOKEN)
 hf_api = HfApi()
 
-# Access the values
-LLM_MODEL_NAME = config["llm_model_name"]
-LLM_MODEL_TEMPERATURE = float(config["llm_model_temperature"])
 
 def initialize():
     global kadiAPY_bot
 
-    vectorstore = get_chroma_vectorstore(get_SFR_Code_embedding_model(), vectorstore_path)
+
+    
+    # download_gitlab_repo_to_hfspace(GITLAB_API_URL, GITLAB_PROJECT_ID, GITLAB_PROJECT_VERSION, DATA_DIR, hf_api, HF_SPACE_NAME)
+    
+    # code_texts, code_references = extract_repo_files(DATA_DIR, ['kadi_apy'], [])
+    # doc_texts, doc_references = extract_repo_files(DATA_DIR, ['docs'], [])
+    
+    # print("Length of code_texts: ", len(code_texts))
+    # print("Length of doc_files: ", len(doc_texts))
+    
+    # code_chunks = chunk_pythoncode_and_add_metadata(code_texts, code_references)
+    # doc_chunks = chunk_text_and_add_metadata(doc_texts, doc_references, CHUNK_SIZE, CHUNK_OVERLAP)
+
+    # print(f"Total number of code_chunks: {len(code_chunks)}")
+    # print(f"Total number of doc_chunks: {len(doc_chunks)}")
+
+    vectorstore = get_chroma_vectorstore(get_SFR_Code_embedding_model(), "data/vectorstore")
     llm = get_groq_llm(LLM_MODEL_NAME, LLM_MODEL_TEMPERATURE, GROQ_API_KEY)
 
     kadiAPY_bot = KadiAPYBot(llm, vectorstore)
 
 initialize()
 
-def bot_kadi(history, state):
-    """
-    Handle user input and generate bot response using gr.State().
-    """
-    user_query = history[-1][0]
-    
-    # Add user query to the bot's state for session-specific history
-    state["history"].append({"query": user_query, "response": None})
-    
-    # Process the query with the bot and generate a response
+
+
+def bot_kadi(history):
+    user_query = history[-1][0]   
     response = kadiAPY_bot.process_query(user_query)
-    
-    # Save the response back to session state
-    state["history"][-1]["response"] = response
-    
     history[-1] = (user_query, response)
-    yield history
-
-# Gradio UI
-def add_text(history, text, state):
-    """
-    Add user text to history and initialize state if needed.
-    """
-    if "history" not in state:
-        state["history"] = []  # Initialize session-specific state
-    if history is None or len(history) == 0:
-    history = []  # Initialize empty history list
 
-    history = history + [(text, None)]
-    yield history, ""
+    yield history  
+
 
+
+# Gradio utils
 def check_input_text(text):
     if not text:
         gr.Warning("Please input a question.")
         raise TypeError
     return True
-    
+
+def add_text(history, text):
+    history = history + [(text, None)]
+    yield history, ""
+
+
+import gradio as gr
+
 def main():
     with gr.Blocks() as demo:
         gr.Markdown("## KadiAPY - AI Coding-Assistant")
@@ -85,9 +109,6 @@ def main():
                 with gr.Column(scale=10):
                     chatbot = gr.Chatbot([], elem_id="chatbot", label="Kadi Bot", bubble_full_width=False, show_copy_button=True, height=600)
                     user_txt = gr.Textbox(label="Question", placeholder="Type in your question and press Enter or click Submit")
-                    
-                    # Create session-specific state with gr.State
-                    session_state = gr.State()  
 
                     with gr.Row():
                         with gr.Column(scale=1):
@@ -108,14 +129,12 @@ def main():
                         examples_per_page=3,
                     )
 
-            user_txt.submit(check_input_text, user_txt, None)
-            .success(add_text, [chatbot, user_txt, session_state], [chatbot, user_txt])
-            .then(bot_kadi, [chatbot, session_state], [chatbot])
-            submit_btn.click(check_input_text, user_txt, None)
-            .success(add_text, [chatbot, user_txt, session_state], [chatbot, user_txt])
-            .then(bot_kadi, [chatbot, session_state], [chatbot])
+            user_txt.submit(check_input_text, user_txt, None).success(add_text, [chatbot, user_txt], [chatbot, user_txt]).then(bot_kadi, [chatbot], [chatbot])
+            submit_btn.click(check_input_text, user_txt, None).success(add_text, [chatbot, user_txt], [chatbot, user_txt]).then(bot_kadi, [chatbot], [chatbot])
             clear_btn.click(lambda: None, None, chatbot, queue=False)
-    demo.launch()
 
+    demo.launch() 
+
+    
 if __name__ == "__main__":
     main()