added semantic search code

app.py

@@ -1,6 +1,71 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
 
+#STEP 1 FROM SEMANTIC SEARCH
+from sentence_transformers import SentenceTransformer
+import torch
+
+#STEP 2 FROM SEMANTIC SEARCH
+with open("water_cycle.txt", "r", encoding="utf-8") as file: # Open the water_cycle.txt file in read mode with UTF-8 encoding
+    water_cycle_text = file.read() # Read file and store into variable
+
+#STEP 3 FROM SEMANTIC SEARCH
+def preprocess_text(text):
+    cleaned_text = text.strip() # Strip extra whitespace from beginning and end of text
+    chunks = cleaned_text.split("\n") # Split cleaned_text by every newline character (\n)
+    cleaned_chunks = [] # Empty list to store cleaned chunks
+
+    for chunk in chunks: # For-in loop to clean each chunk and add to list
+        chunk = chunk.strip()
+        if chunk != "":
+            cleaned_chunks.append(chunk)
+        
+    print(cleaned_chunks)
+    print(len(cleaned_chunks))
+    
+    return cleaned_chunks
+
+cleaned_chunks = preprocess_text(water_cycle_text) # Call preprocess_text and store result in cleaned_chunks
+
+#STEP 4 FROM SEMANTIC SEARCH
+model = SentenceTransformer('all-MiniLM-L6-v2') # Load pre-trained embedding model that converts text to vectors
+
+def create_embeddings(text_chunks):
+    chunk_embeddings = model.encode(text_chunks, convert_to_tensor=True) # Convert each text chunk into a vector embedding and store as a tensor
+
+    print(chunk_embeddings)
+    print(chunk_embeddings.shape)
+
+    return chunk_embeddings
+
+chunk_embeddings = create_embeddings(cleaned_chunks) # Call create_embeddings and store result in chunk_embeddings
+
+#STEP 5 FROM SEMANTIC SEARCH
+def get_top_chunks(query, chunk_embeddings, text_chunks): #Finds most relevant text chunks for given query, chunk_embeddings, and text_chunks
+    query_embedding = model.encode(query,convert_to_tensor=True) # Convert query string into vector embedding
+
+    query_embedding_normalized = query_embedding / query_embedding.norm() # Normalize query embedding to unit length for accurate similarity comparison
+    chunk_embeddings_normalized = chunk_embeddings / chunk_embeddings.norm(dim=1, keepdim=True) # Normalize all chunk embeddings to unit length for consistent comparison
+
+    similarities = torch.matmul(chunk_embeddings_normalized,query_embedding_normalized) # Calculate cosine similarity between query and all chunks using matrix multiplication
+    print(similarities)
+
+    top_indices = torch.topk(similarities, k=3).indices # Find indices of the 3 chunks with highest similarity scores
+    print(top_indices)
+
+    top_chunks = [] # Empty list to store most relevant chunks
+
+    for i in top_indices: # Loop through top indices to retrieve corresponding text chunks
+        chunk=text_chunks[i]
+        top_chunks.append(chunk)
+
+    return top_chunks
+
+#STEP 6 FROM SEMANTIC SEARCH
+top_results = get_top_chunks("How does the water cycle work?", chunk_embeddings, cleaned_chunks) # Call get_top_chunks with query
+print(top_results)
+
+#SAMPLE HUGGING FACE PROJECT
 client = InferenceClient("Qwen/Qwen2.5-72B-instruct")
 
 def respond(message, history):