app redone

app.py

@@ -1,107 +1,90 @@
-# Install neccessary requirements
-pip install requirments.txt
+# Import necessary libraries
 
-# Import neccessary libraries
 import streamlit as st
 import pandas as pd
 import numpy as np
 import requests
-import torch
-from tqdm.auto import tqdm
-from transformers import BertModel, BertTokenizer
+from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 
-# CourseFAQBot class
-class CourseFAQBot:
-    def __init__(self, model_name="bert-base-uncased", docs_url=None, batch_size=8):
-        self.tokenizer = BertTokenizer.from_pretrained(model_name)
-        self.model = BertModel.from_pretrained(model_name)
-        self.model.eval()  # Set the model to evaluation mode if not training
-        self.batch_size = batch_size
-        self.df = self._download_and_process_documents(docs_url)
-        self.document_embeddings = self.compute_embeddings(self.df['text'].tolist())
+# Function to fetch data
+def fetch_data():
+    docs_url = 'https://github.com/alexeygrigorev/llm-rag-workshop/raw/main/notebooks/documents.json'
+    docs_response = requests.get(docs_url)
+    documents_raw = docs_response.json()
+    documents = []
 
-    def _download_and_process_documents(self, docs_url):
-        """
-        Download and process the document data.
-        """
-        docs_response = requests.get(docs_url)
-        documents_raw = docs_response.json()
-        
-        documents = []
-        for course in documents_raw:
-            course_name = course['course']
-            for doc in course['documents']:
-                doc['course'] = course_name
-                documents.append(doc)
-        
-        # Create the DataFrame
-        return pd.DataFrame(documents, columns=['course', 'section', 'question', 'text'])
+    for course in documents_raw:
+        course_name = course['course']
+        for doc in course['documents']:
+            doc['course'] = course_name
+            documents.append(doc)
+    
+    return pd.DataFrame(documents, columns=['course', 'section', 'question', 'text'])
+
+# TextSearch class
+class TextSearch:
+    def __init__(self, text_fields):
+        self.text_fields = text_fields
+        self.matrices = {}
+        self.vectorizers = {}
 
-    def make_batches(self, seq, n):
-        """
-        Split a sequence into batches of size n.
-        """
-        result = []
-        for i in range(0, len(seq), n):
-            batch = seq[i:i+n]
-            result.append(batch)
-        return result
+    def fit(self, records, vectorizer_params={}):
+        self.df = pd.DataFrame(records)
+        for f in self.text_fields:
+            cv = TfidfVectorizer(**vectorizer_params)
+            X = cv.fit_transform(self.df[f])
+            self.vectorizers[f] = cv
+            self.matrices[f] = X
 
-    def compute_embeddings(self, texts):
-        """
-        Compute embeddings for a list of texts using a pre-trained transformer model.
-        """
-        text_batches = self.make_batches(texts, self.batch_size)
-        all_embeddings = []
+    def search(self, query, filters={}, boost={}):
+        score = np.zeros(len(self.df))
+        for f in self.text_fields:
+            b = boost.get(f, 1.0)
+            q = self.vectorizers[f].transform([query])
+            s = cosine_similarity(self.matrices[f], q).flatten()
+            score = score + b * s
         
-        for batch in tqdm(text_batches, desc="Computing embeddings"):
-            encoded_input = self.tokenizer(batch, padding=True, truncation=True, return_tensors='pt')
-            with torch.no_grad():
-                outputs = self.model(**encoded_input)
-                hidden_states = outputs.last_hidden_state
-                batch_embeddings = hidden_states.mean(dim=1)
-                batch_embeddings_np = batch_embeddings.cpu().numpy()
-                all_embeddings.append(batch_embeddings_np)
+        for field, value in filters.items():
+            mask = (self.df[field] == value).values
+            score = score * mask
         
-        final_embeddings = np.vstack(all_embeddings)
-        return final_embeddings
+        idx = np.argsort(-score)[:5]
+        return self.df.iloc[idx].to_dict(orient='records')
 
-    def query(self, query_text, top_n=10):
-        """
-        Perform a query to find the most relevant documents.
-        """
-        query_embedding = self.compute_embeddings([query_text])
-        similarities = cosine_similarity(query_embedding, self.document_embeddings).flatten()
-        top_n_indices = similarities.argsort()[-top_n:][::-1]
-        top_n_documents = self.df.iloc[top_n_indices]
-        return top_n_documents
-
-# Streamlit application
+# Main Streamlit application
 st.title("FAQ Search Engine for DataTalks")
 
-# Initialize CourseFAQBot
-docs_url = 'https://github.com/alexeygrigorev/llm-rag-workshop/raw/main/notebooks/documents.json'
-faq_bot = CourseFAQBot(docs_url=docs_url)
+# Load data
+df = fetch_data()
+
+# Initialize TextSearch
+text_search = TextSearch(text_fields=['section', 'question', 'text'])
+text_search.fit(df.to_dict(orient='records'), vectorizer_params={'stop_words': 'english', 'min_df': 3})
 
 # Input fields for query and filters
 query = st.text_input("Enter your query:")
-courses = st.multiselect("Select course(s):", options=faq_bot.df['course'].unique())
+courses = st.multiselect("Select course(s):", options=df['course'].unique())
 
 # Search button
 if st.button("Search"):
-    results = faq_bot.query(query)
-    
-    # Filter results by selected courses if any
+    filters = {}
     if courses:
-        results = results[results['course'].isin(courses)]
+        filters['course'] = courses[0] if len(courses) == 1 else courses
+    results = text_search.search(query, filters=filters, boost={'question': 3.0})
     
-    # Display results with space in between
-    for i, result in enumerate(results.to_dict(orient='records')):
+    # Display results
+    # for i, result in enumerate(results):
+    #     st.write(f"### Result {i+1}")
+    #     st.write(f"**Course**: {result['course']}")
+    #     st.write(f"**Question**: {result['question']}")
+    #     st.write(f"**Response**: {result['text']}")
+
+    for i, result in enumerate(results):
         st.write(f"### Result {i+1}")
         st.write(f"**Course**: {result['course']}")
         st.write(f"**Section**: {result['section']}")
         st.write(f"**Question**: {result['question']}")
         st.write(f"**Text**: {result['text']}")
-        st.write("")  # Adds a blank space between results
-        st.markdown("---")
+        st.write("")  
+        st.markdown("---")