Create app.py

app.py

@@ -0,0 +1,82 @@
+import streamlit as st
+import pickle
+import os
+import json
+
+from collections import defaultdict
+from langchain.vectorstores import FAISS
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from rank_bm25 import BM25Okapi
+
+# Constants
+BASE_DIR = "built_index"
+VECTOR_STORE_DIR = os.path.join(BASE_DIR, "vector_store")
+BM25_INDEX_FILE = os.path.join(BASE_DIR, "bm25_index.pkl")
+SEARCH_INDEX_FILE = os.path.join(BASE_DIR, "search_index.json")
+
+# Load embedding model
+@st.cache_resource
+def load_embeddings():
+    return HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+# Load indexes
+@st.cache_resource
+def load_indexes():
+    # Load search index
+    with open(SEARCH_INDEX_FILE, "r") as f:
+        index = defaultdict(dict, json.load(f))
+
+    # Load vector store
+    embeddings = load_embeddings()
+    vector_store = FAISS.load_local(VECTOR_STORE_DIR, embeddings, allow_dangerous_deserialization=True)
+
+    # Load BM25 index
+    with open(BM25_INDEX_FILE, "rb") as f:
+        bm25, bm25_texts, url_order = pickle.load(f)
+
+    return index, vector_store, bm25, bm25_texts, url_order
+
+# Search functions
+def semantic_search(vector_store, query, k=5):
+    results = vector_store.similarity_search(query, k=k)
+    return [{
+        "url": r.metadata.get("url", "N/A"),
+        "snippet": r.page_content[:200]
+    } for r in results]
+
+def bm25_search(bm25, bm25_texts, url_order, index, query, k=5):
+    query_tokens = query.lower().split()
+    scores = bm25.get_scores(query_tokens)
+    top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:k]
+    return [{
+        "url": url_order[i],
+        "score": scores[i],
+        "snippet": index[url_order[i]]["content"][:200]
+    } for i in top_indices]
+
+# Streamlit UI
+def main():
+    st.set_page_config(page_title="LangChain Search Engine", layout="wide")
+    st.title("🔍 LangChain Search Engine (Semantic + BM25)")
+
+    query = st.text_input("Enter your search query:", "")
+
+    if query:
+        index, vector_store, bm25, bm25_texts, url_order = load_indexes()
+        
+        with st.spinner("Searching..."):
+            sem_results = semantic_search(vector_store, query)
+            bm25_results = bm25_search(bm25, bm25_texts, url_order, index, query)
+
+        st.subheader("🔎 Semantic Search Results")
+        for i, res in enumerate(sem_results, 1):
+            st.markdown(f"**{i}. [{res['url']}]({res['url']})**")
+            st.write(res['snippet'] + "...")
+
+        st.subheader("🧮 BM25 Sparse Search Results")
+        for i, res in enumerate(bm25_results, 1):
+            st.markdown(f"**{i}. [{res['url']}]({res['url']})** (Score: {res['score']:.2f})")
+            st.write(res['snippet'] + "...")
+
+if __name__ == "__main__":
+    main()