Spaces:
Build error
Build error
File size: 3,453 Bytes
d0a9049 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | import streamlit as st
import numpy as np
import pandas as pd
from langchain.chains import RetrievalQA
from langchain_community.embeddings import SentenceTransformerEmbeddings
from langchain_community.vectorstores import FAISS
from langchain.agents import initialize_agent, Tool, AgentType
from langchain.prompts import PromptTemplate
import faiss
import os
# Here i Loaded my preprocessed data into a DataFrame (CSV)
@st.cache_data
def load_data():
# Collected the data and stored in CSV file with columns: 'Course_Title', 'Description', 'Link', 'Duration', 'Instructor'
df = pd.read_csv("free_courses.csv", encoding="ISO-8859-1") # or 'latin1', 'cp1252'
return df
# Step 1: Preprocessing (Optional) - Clean and tokenize the data
def preprocess_data(df):
df['description'] = df['Description'].str.lower()
#df['Link'] = df['Link'].str.lower()
return df
# Step 2: Create Embeddings using Sentence-BERT
def generate_embeddings(df):
# Instantiate the Sentence-BERT embedding model
embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
embeddings = embedding_model.embed_documents(df['description'].tolist())
return embeddings
# Step 3: Create Vector Database (FAISS)
def create_vector_db(embeddings):
# Convert the embeddings to a numpy array and use FAISS for fast similarity search
embeddings_np = np.array(embeddings)
dimension = embeddings_np.shape[1] # Embedding dimension (e.g., 384 for all-MiniLM-L6-v2)
# Initialize FAISS index
index = faiss.IndexFlatL2(dimension) # L2 distance for similarity search
index.add(embeddings_np) # Add the embeddings to the FAISS index
return index
# Step 4: Search Functionality with LangChain's Retrieval
def search_courses(query, index, df):
# Generate the query embedding
embedding_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
query_embedding = embedding_model.embed_query(query)
# Convert the query to numpy array for FAISS search
query_embedding_np = np.array(query_embedding).reshape(1, -1)
# Perform the search using FAISS
D, I = index.search(query_embedding_np, k=3) # Top 5 results
results = df.iloc[I[0]] # Get the corresponding courses
return results
# Step 5: Display Results in Streamlit
def display_results(results):
for idx, row in results.iterrows():
st.write(f"### [{row['Course_Title']}]({row['Link']})")
st.write(f"**Description:** {row['Description']}")
#st.write(f"**Link:** {row['Link']}")
st.write(f"**Duration:** {row['Duration']}")
st.write(f"**Instructor:** {row['Instructor']}")
st.write("---")
# Step 6: Setup the Streamlit Interface
def main():
# Load and preprocess data
df = load_data()
df = preprocess_data(df)
# Generate course embeddings and create a FAISS vector database
embeddings = generate_embeddings(df)
index = create_vector_db(embeddings)
# Set up Streamlit UI
st.title("Smart Search Tool for Free Courses on Analytics Vidhya")
query = st.text_input("Enter your search query:", "")
if query:
results = search_courses(query, index, df)
display_results(results)
else:
st.write("Please enter a search query to find relevant courses.")
if __name__ == "__main__":
main()
|