import streamlit as st import numpy as np import pandas as pd from langchain.chains import RetrievalQA from langchain_community.embeddings import SentenceTransformerEmbeddings from langchain_community.vectorstores import FAISS from langchain.agents import initialize_agent, Tool, AgentType from langchain.prompts import PromptTemplate import faiss import os # Here i Loaded my preprocessed data into a DataFrame (CSV) @st.cache_data def load_data(): # Collected the data and stored in CSV file with columns: 'Course_Title', 'Description', 'Link', 'Duration', 'Instructor' df = pd.read_csv("free_courses.csv", encoding="ISO-8859-1") # or 'latin1', 'cp1252' return df # Step 1: Preprocessing (Optional) - Clean and tokenize the data def preprocess_data(df): df['description'] = df['Description'].str.lower() #df['Link'] = df['Link'].str.lower() return df # Step 2: Create Embeddings using Sentence-BERT def generate_embeddings(df): # Instantiate the Sentence-BERT embedding model embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") embeddings = embedding_model.embed_documents(df['description'].tolist()) return embeddings # Step 3: Create Vector Database (FAISS) def create_vector_db(embeddings): # Convert the embeddings to a numpy array and use FAISS for fast similarity search embeddings_np = np.array(embeddings) dimension = embeddings_np.shape[1] # Embedding dimension (e.g., 384 for all-MiniLM-L6-v2) # Initialize FAISS index index = faiss.IndexFlatL2(dimension) # L2 distance for similarity search index.add(embeddings_np) # Add the embeddings to the FAISS index return index # Step 4: Search Functionality with LangChain's Retrieval def search_courses(query, index, df): # Generate the query embedding embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") query_embedding = embedding_model.embed_query(query) # Convert the query to numpy array for FAISS search query_embedding_np = np.array(query_embedding).reshape(1, -1) # Perform the search using FAISS D, I = index.search(query_embedding_np, k=3) # Top 5 results results = df.iloc[I[0]] # Get the corresponding courses return results # Step 5: Display Results in Streamlit def display_results(results): for idx, row in results.iterrows(): st.write(f"### [{row['Course_Title']}]({row['Link']})") st.write(f"**Description:** {row['Description']}") #st.write(f"**Link:** {row['Link']}") st.write(f"**Duration:** {row['Duration']}") st.write(f"**Instructor:** {row['Instructor']}") st.write("---") # Step 6: Setup the Streamlit Interface def main(): # Load and preprocess data df = load_data() df = preprocess_data(df) # Generate course embeddings and create a FAISS vector database embeddings = generate_embeddings(df) index = create_vector_db(embeddings) # Set up Streamlit UI st.title("Smart Search Tool for Free Courses on Analytics Vidhya") query = st.text_input("Enter your search query:", "") if query: results = search_courses(query, index, df) display_results(results) else: st.write("Please enter a search query to find relevant courses.") if __name__ == "__main__": main()