app.py

import streamlit as st
import numpy as np
import pandas as pd
from langchain.chains import RetrievalQA
from langchain_community.embeddings import SentenceTransformerEmbeddings
from langchain_community.vectorstores import FAISS
from langchain.agents import initialize_agent, Tool, AgentType
from langchain.prompts import PromptTemplate
import faiss
import os

# Here i Loaded my preprocessed data into a DataFrame (CSV)
@st.cache_data
def load_data():
    # Collected the data and stored in CSV file with columns: 'Course_Title', 'Description', 'Link', 'Duration', 'Instructor'
    df = pd.read_csv("free_courses.csv", encoding="ISO-8859-1")  # or 'latin1', 'cp1252'
    return df

# Step 1: Preprocessing (Optional) - Clean and tokenize the data
def preprocess_data(df):
    df['description'] = df['Description'].str.lower()
    #df['Link'] = df['Link'].str.lower()
    return df

# Step 2: Create Embeddings using Sentence-BERT
def generate_embeddings(df):
    # Instantiate the Sentence-BERT embedding model
    embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
    embeddings = embedding_model.embed_documents(df['description'].tolist())
    return embeddings

# Step 3: Create Vector Database (FAISS)
def create_vector_db(embeddings):
    # Convert the embeddings to a numpy array and use FAISS for fast similarity search
    embeddings_np = np.array(embeddings)
    dimension = embeddings_np.shape[1]  # Embedding dimension (e.g., 384 for all-MiniLM-L6-v2)
    
    # Initialize FAISS index
    index = faiss.IndexFlatL2(dimension)  # L2 distance for similarity search
    index.add(embeddings_np)  # Add the embeddings to the FAISS index
    return index

# Step 4: Search Functionality with LangChain's Retrieval
def search_courses(query, index, df):
    # Generate the query embedding
    embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
    query_embedding = embedding_model.embed_query(query)
    
    # Convert the query to numpy array for FAISS search
    query_embedding_np = np.array(query_embedding).reshape(1, -1)
    
    # Perform the search using FAISS
    D, I = index.search(query_embedding_np, k=3)  # Top 5 results
    results = df.iloc[I[0]]  # Get the corresponding courses
    return results

# Step 5: Display Results in Streamlit
def display_results(results):
    for idx, row in results.iterrows():
        st.write(f"### [{row['Course_Title']}]({row['Link']})")
        st.write(f"**Description:** {row['Description']}")
        #st.write(f"**Link:** {row['Link']}")
        st.write(f"**Duration:** {row['Duration']}")
        st.write(f"**Instructor:** {row['Instructor']}")
        st.write("---")

# Step 6: Setup the Streamlit Interface
def main():
    # Load and preprocess data
    df = load_data()
    df = preprocess_data(df)
    
    # Generate course embeddings and create a FAISS vector database
    embeddings = generate_embeddings(df)
    index = create_vector_db(embeddings)
    
    # Set up Streamlit UI
    st.title("Smart Search Tool for Free Courses on Analytics Vidhya")
    query = st.text_input("Enter your search query:", "")
    
    if query:
        results = search_courses(query, index, df)
        display_results(results)
    else:
        st.write("Please enter a search query to find relevant courses.")

if __name__ == "__main__":
    main()