Update app.py

app.py

@@ -1,5 +1,5 @@
 import streamlit as st
-from gensim.models import FastText
+from gensim.models import FastText, KeyedVectors
 import re
 from gensim.utils import simple_preprocess
 import time
@@ -9,6 +9,8 @@ import io
 import tempfile
 import numpy as np
 from concurrent.futures import ThreadPoolExecutor
+from huggingface_hub import hf_hub_download
+from sklearn.metrics.pairwise import cosine_similarity
 
 # Function to preprocess text
 def preprocess_text(text):
@@ -65,14 +67,61 @@ def clean_text_multithreaded(text):
         cleaned_chunks = list(executor.map(clean_text_chunk, chunks))
     return '\n'.join(cleaned_chunks)
 
+# Function to load the FastText model from Hugging Face
+@st.cache_resource
+def load_fasttext_model(model_dir):
+    model_path = os.path.join(model_dir, "fasttext_model.model")
+    vectors_path = os.path.join(model_dir, "fasttext_model_vectors.kv")
+    vectors_ngrams_path = os.path.join(model_dir, "fasttext_model.model.wv.vectors_ngrams.npy")
+    
+    model = FastText.load(model_path)
+    model.wv = KeyedVectors.load(vectors_path, mmap='r')
+    model.wv.vectors_ngrams = np.load(vectors_ngrams_path, mmap_mode='r')
+    
+    return model
+
+
+# Function to generate embeddings for a given word
+def generate_word_embedding(word, model):
+    return model.wv.get_vector(word, norm=True) if word in model.wv else None
+
+# Function to find similar words
+def find_similar_words(word, model, topn=5):
+    return model.wv.most_similar(word, topn=topn) if word in model.wv else []
+
+# Function to tokenize a sentence using the given pattern
+def tokenize_sentence(sentence, pattern):
+    tokens = re.findall(pattern, sentence)
+    return [token.strip() for token in tokens if token.strip()]
+
+# Function to generate embeddings for words in a sentence
+def generate_embeddings_for_sentence(sentence, model, pattern):
+    tokens = tokenize_sentence(sentence, pattern)
+    embeddings = []
+    for token in tokens:
+        if token in model.wv:
+            embeddings.append(model.wv[token])
+    return embeddings
+
+# Function to generate embedding for a sentence
+def generate_sentence_embedding(sentence, model, pattern):
+    word_embeddings = generate_embeddings_for_sentence(sentence, model, pattern)
+    if not word_embeddings:
+        return None
+    return np.mean(word_embeddings, axis=0)
+
+# Function to generate embeddings for sentences
+def generate_sentence_embeddings(sentences, model, pattern):
+    return [generate_sentence_embedding(sentence, model, pattern) for sentence in sentences]
+
 # Streamlit app
 def main():
     st.title("Text Processing and FastText Word Embedding Trainer")
     
     # Sidebar options
     st.sidebar.title("Options")
-    option = st.sidebar.radio("Select an option", ("Clean Dataset", "Train Word Embedding"))
-    
+    option = st.sidebar.radio("Select an option", ("Clean Dataset", "Train Word Embedding", "Generate Embeddings"))
+
     if option == "Clean Dataset":
         st.header("Clean Text Dataset")
         
@@ -153,6 +202,74 @@ def main():
                 except Exception as e:
                     st.error(f"An error occurred: {str(e)}")
                     st.error("Check the server logs for more details.")
+    
+    elif option == "Generate Embeddings":
+        st.header("Generate Embeddings with Pretrained FastText Model")
+        
+        repo_id = "Blessmore/Fasttext_embeddings/Fast_text_50_dim"
+        model_file = "fasttext_model.model"
+        vectors_file = "fasttext_model_vectors.kv"
+        vectors_ngrams_file = "fasttext_model.model.wv.vectors_ngrams.npy"
+
+        model = load_fasttext_model(repo_id, model_file, vectors_file, vectors_ngrams_file)
+
+        st.subheader("Generate Word Embedding")
+        word = st.text_input("Enter a word:")
+        if word:
+            embedding = generate_word_embedding(word, model)
+            if embedding is not None:
+                st.write(f"Embedding for '{word}':", embedding)
+            else:
+                st.write(f"'{word}' not in vocabulary")
+
+        st.subheader("Find Similar Words")
+        word_for_similar = st.text_input("Enter a word to find similar words:")
+        if word_for_similar:
+            similar_words = find_similar_words(word_for_similar, model)
+            if similar_words:
+                st.write("Similar words:")
+                for word, similarity in similar_words:
+                    st.write(f"{word}: {similarity}")
+            else:
+                st.write(f"No similar words found for '{word_for_similar}'")
+
+        st.subheader("Generate Embeddings for Words in a Sentence")
+        sentence = st.text_input("Enter a sentence:")
+        if sentence:
+            word_embeddings = generate_embeddings_for_sentence(sentence, model, r'\b\w+\b')
+            if word_embeddings:
+                for idx, embedding in enumerate(word_embeddings):
+                    st.write(f"Word {idx+1} embedding:", embedding)
+            else:
+                st.write("No embeddings could be generated for the words in the sentence.")
+
+        st.subheader("Generate Embedding for a Sentence")
+        sentence_for_embedding = st.text_input("Enter a sentence to generate its embedding:")
+        if sentence_for_embedding:
+            sentence_embedding = generate_sentence_embedding(sentence_for_embedding, model, r'\b\w+\b')
+            if sentence_embedding is not None:
+                st.write("Sentence embedding:", sentence_embedding)
+            else:
+                st.write("No embedding could be generated for the sentence.")
+
+        st.subheader("Find Most Similar Sentence Pairs")
+        uploaded_sentences_file = st.file_uploader("Upload a text file with sentences (one per line)", type=["txt"])
+        if uploaded_sentences_file:
+            sentences = uploaded_sentences_file.read().decode('utf-8').splitlines()
+            sentence_embeddings = generate_sentence_embeddings(sentences, model, r'\b\w+\b')
+            sentence_pairs = []
+            for i in range(len(sentences)):
+                for j in range(i + 1, len(sentences)):
+                    if sentence_embeddings[i] is not None and sentence_embeddings[j] is not None:
+                        similarity = cosine_similarity([sentence_embeddings[i]], [sentence_embeddings[j]])[0][0]
+                        sentence_pairs.append((sentences[i], sentences[j], similarity))
+            sentence_pairs = sorted(sentence_pairs, key=lambda x: x[2], reverse=True)
+            st.write("Most similar sentence pairs:")
+            for sent1, sent2, sim in sentence_pairs[:5]:
+                st.write(f"Sentence 1: {sent1}")
+                st.write(f"Sentence 2: {sent2}")
+                st.write(f"Similarity: {sim}")
+                st.write("-----")
 
 if __name__ == "__main__":
     main()