Add full quotation data

app.py

@@ -40,7 +40,7 @@ if st.button("Search Quotes"):
         # Format and display search results
         st.write(f"Search Results: ")
         for i, (prompt, distance) in enumerate(zip(similar_quotes, distances)):
-            st.write(f"{i+1}. Prompt: {prompt}, Distance: {distance}")
+            st.write(f"{i+1}. Quote: {prompt}, Cosine similarity: {distance}")
             print(f'Those are: {prompt}, {distance}')
     else:
         st.error("Please enter a quote or phrase.")

fast_api.py

@@ -32,8 +32,8 @@ def read_root():
 async def search_prompts(query: Query, k: int = 3):
     print(f'Prompt: {query}')
     similar_quotes, distances = search_engine.most_similar(query.quote, top_k=k)
-    print(f'Similar Quotes {similar_quotes}')
-    print(f'Distances {distances}')
+    print(f'Similar Quotes: {similar_quotes}')
+    print(f'Cosine similarity: {distances}')
     print(40*'****')
     # Format the response
     response = [
@@ -48,7 +48,7 @@ async def all_vectors(query: Query):
 
     query_embedding = search_engine.model.encode([query.quote])  # Encode the prompt to a vector
     all_similarities = search_engine.cosine_similarity(query_embedding, search_engine.index)
-    print(f'Prompt: {query}')
+    print(f'Quote or phrase: {query}')
     print(f'All Vector Similarities: {all_similarities}')
     print(40*'****')
     response = [

models/data_reader.py

@@ -2,6 +2,7 @@ from datasets import load_dataset
 import json
 from tqdm import tqdm
 import pandas as pd
+# import kagglehub
 
 
 
@@ -59,7 +60,7 @@ def load_quotes_from_csv(file_path):
     print("Quotes loaded:", len(quotes))   # should be 499709
     print("First quote:", quotes[0][:100])
     print("Data loaded successfully.")
-    return quotes[:10000]
+    return quotes
 
 
 if __name__ == "__main__":

models/quotes_search_engine.py

@@ -20,7 +20,7 @@ class QuoteSearchEngine:
         print("Data encoding started...")
         print("Number of quotes to encode:", len(quotes))
    
-        batch_size = 1000
+        batch_size = 10000
         for i in range(0, len(quotes), batch_size):
             batch = quotes[i:i+batch_size]
             embeddings = self.model.encode(batch, batch_size=128, show_progress_bar=True)
@@ -44,7 +44,7 @@ class QuoteSearchEngine:
         # Retrieve the corresponding prompts for the found indices
         similar_prompts = [self.prompts_track[idx] for idx in indices[0]]
         
-        return similar_prompts, distances[0]  # Return both the similar prompts and their distances
+        return similar_prompts, round(distances[0], 3)  # Return both the similar prompts and their distances
 
 
     def cosine_similarity(self, query_vector, index):