Upload app.py

app.py

@@ -1,7 +1,10 @@
 import os
+import re
 import zipfile
 import gradio as gr
-
+from langchain_community.vectorstores import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import FAISS
@@ -10,16 +13,41 @@ from langchain.chains import ConversationalRetrievalChain
 from langchain.memory import ConversationBufferMemory
 from langchain.prompts import PromptTemplate
 
-# --- Step 1: Unzip FAISS index ---
-if not os.path.exists("faiss_index") and os.path.exists("faiss_index.zip"):
-    with zipfile.ZipFile("faiss_index.zip", "r") as zip_ref:
-        zip_ref.extractall(".")
-
-# --- Step 2: Load embedding and vectorstore ---
-embedding_model = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
-vectordb = FAISS.load_local("faiss_index", embedding_model)
-
-# --- Step 3: Load the LLM ---
+# --- Step 1: Clean .txt files ---
+DATA_DIR = "knowledge_base"
+docs = []
+
+for fname in os.listdir(DATA_DIR):
+    file_path = os.path.join(DATA_DIR, fname)
+    if os.path.isfile(file_path) and fname.endswith(".txt"):
+        with open(file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+            cleaned_text = text.replace('\xa0', ' ')
+            cleaned_text = re.sub(r'\n+', '\n', cleaned_text)
+            cleaned_text = re.sub(r' +', ' ', cleaned_text).strip()
+            docs.append({"page": fname, "text": cleaned_text})
+
+# --- Step 2: Split text into chunks ---
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+texts = []
+metadatas = []
+
+for doc in docs:
+    chunks = splitter.split_text(doc["text"])
+    for i, chunk in enumerate(chunks):
+        texts.append(chunk)
+        metadatas.append({"source": doc["page"], "chunk": i})
+
+# --- Step 3: Create Document objects ---
+documents = [Document(page_content=texts[i], metadata=metadatas[i]) for i in range(len(texts))]
+
+# --- Step 4: Load embedding model ---
+embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+# --- Step 5: Build FAISS index ---
+vectordb = FAISS.from_documents(documents, embedding_model)
+
+# --- Step 6: Load the LLM ---
 model_id = "tiiuae/falcon3-1b-instruct"
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(model_id)
@@ -35,7 +63,7 @@ pipe = pipeline(
 )
 llm = HuggingFacePipeline(pipeline=pipe)
 
-# --- Step 4: Setup memory and QA chain ---
+# --- Step 7: Setup memory and QA chain ---
 memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
 
 prompt = PromptTemplate.from_template("""
@@ -59,7 +87,7 @@ qa_chain = ConversationalRetrievalChain.from_llm(
     combine_docs_chain_kwargs={"prompt": prompt}
 )
 
-# --- Step 5: Define chatbot logic ---
+# --- Step 8: Define chatbot logic ---
 def chat(message, history):
     result = qa_chain.invoke({"question": message})
     response = result.get("answer", "")
@@ -69,7 +97,7 @@ def chat(message, history):
 # UH logo
 UH_LOGO = "images/UH.png"
 
-# --- Step 6: UI ---
+# --- Step 9: UI ---
 sample_questions = [
     "How do I register as a new student?",
     "Where can I find accommodation?",