updated

app.py

@@ -1,17 +1,3 @@
-import subprocess
-
-# Run the pip install command for pyenchant
-subprocess.run(["pip", "install", "pyenchant"], check=True)
-
-# # Run the first command
-# subprocess.run(["apt", "install", "enchant", "--fix-missing", "-y"], check=True)
-
-# # Run the second command
-# subprocess.run(["apt", "install", "-qq", "enchant", "-y"], check=True)
-
-# # Run the pip install command for pyenchant
-# subprocess.run(["pip", "install", "pyenchant"], check=True)
-
 import streamlit as st
 import pandas as pd
 import pickle
@@ -21,24 +7,31 @@ import joblib
 from googletrans import Translator
 from indictrans import Transliterator
 from pyphonetics import RefinedSoundex
-import enchant
 from bs4 import BeautifulSoup
 import re
 
+def closest_match(word, vocabulary):
+    best_match = None
+    best_distance = float('inf')
+    for vocab_word in vocabulary:
+        dist = lev(word, vocab_word)
+        if dist < best_distance:
+            best_distance = dist
+            best_match = vocab_word
+    return best_match
+
 def main():
     st.title('Text Processing App')
-
-    dictn = enchant.Dict("en_US")
     rs = RefinedSoundex()
     normalized_string_final=[]
     translator = Translator()
     trn = Transliterator(source='eng', target='hin')
 
     with open(r'./english_vocab.pkl', "rb") as fp:
-       english = pickle.load(fp)
+        english = pickle.load(fp)
     english_vocab=english 
     with open(r'./hinglish_vocab.pkl', "rb") as fp:
-       hinglish = pickle.load(fp)
+        hinglish = pickle.load(fp)
     hinglish_vocab=hinglish 
 
     english_vocab['and'] = ['and']
@@ -57,241 +50,146 @@ def main():
     input_text = st.text_area("Enter the text:")
     total_translated = []
     if st.button('Process'):
-        # Create a DataFrame with the user input text
         data = {'Text': [input_text]}
         df1 = pd.DataFrame(data)
-
-        # Apply the clean_tweet function to the user input text
         df1['Text'] = df1['Text'].apply(clean_tweet)
-
-        # Extract the cleaned text
         cleaned_text = df1['Text'].tolist()[0]
-
-        # Process the cleaned text further if needed
         total_text = [cleaned_text]
         st.write("Input Text:", total_text)
-        
+
         for i in tqdm(total_text):
             test_text=i.split()
-
-            # english word change from vocab
             not_changed_idx=[]
             for i in range(len(test_text)):
                 not_changed_idx.append(0)
-            
             changed_text=[]
             changed_idx=[]
-        #     print("1st",changed_text)
-            for i in range(len(test_text)):
 
+            for i in range(len(test_text)):
                 for key in english_vocab:
                     done=0
                     for val in  english_vocab[key]:
                         if(test_text[i]==val):
-                            # print("KEY = ",key,"VAL =",val,"i =",test_text[i],"ADJENCENCY_DATA =",adjacency_data[key])
-        #                     print("yahan par",key,val,test_text[i])
                             changed_text.append(key)
                             changed_idx.append(i)
                             not_changed_idx[i]=1
                             done=1
-                            # print("breaking")
                             break
                     if done==1:
-                        # print("breaking again")
                         break
-                
-            normalized_string=[]
 
-            # making changed text and idx to a dictionary with two lists
+
+            normalized_string=[]
             res = dict(zip(changed_idx, changed_text))
-        #     print(res)
             for i in range(len(test_text)):
                 try:
                     normalized_string.append(res[i])
                 except:
                     normalized_string.append(test_text[i])
-            print("English Normalized String : ",normalized_string)
-
+            print("English Normalized String:", normalized_string)
 
             # hinglish word change
             test_list = [i for i in range(len(test_text))]
             changed_hing_idx = [i for i in test_list if i not in changed_idx]
-            # print(changed_hing_idx)
-            hinglish_text_part=[]
-            for i in changed_hing_idx:
-                try:
-                    hinglish_text_part.append(test_text[i])
-                except:
-                    pass
-        #     print(hinglish_text_part)
+            hinglish_text_part = [test_text[i] for i in changed_hing_idx]
+            changed_text2 = []
+            changed_idx2 = []
 
-            changed_text2=[]
-            changed_idx2=[]
-        #     print("1st hing",changed_text2)
             for i in range(len(hinglish_text_part)):
-
                 for key in hinglish_vocab:
-                    done=0
-                    for val in  hinglish_vocab[key]:
-                        if(hinglish_text_part[i]==val):
-                            # print("KEY = ",key,"VAL =",val,"i =",test_text[i],"ADJENCENCY_DATA =",adjacency_data[key])
-        #                     print(key,val,hinglish_text_part[i])
+                    done = 0
+                    for val in hinglish_vocab[key]:
+                        if hinglish_text_part[i] == val:
                             changed_text2.append(key)
                             changed_idx2.append(i)
-                            not_changed_idx[i]=1
-                            done=1
-                            # print("breaking")
+                            done = 1
                             break
-                    if done==1:
-                        # print("breaking again")
+                    if done == 1:
                         break
 
-
-            # making changed text and idx to a dictionary with two lists
-            normalized_string2=[]
-        #     print("changed_text 2 ",changed_text2)
+            normalized_string2 = []
             res2 = dict(zip(changed_idx2, changed_text2))
-        #     print(res2)
             for i in range(len(hinglish_text_part)):
                 try:
                     normalized_string2.append(res2[i])
                 except:
                     normalized_string2.append(hinglish_text_part[i])
-        #     print("normalised string 2 :",normalized_string2)
-
 
-            changed_idx=list(set(changed_idx))
-            changed_idx.sort()
-        #     print("changed idx",changed_idx)
             for i in changed_idx:
                 normalized_string2.append(res[i])
 
-            print("Hinglish Normalized String : ",normalized_string)
-        #     print(not_changed_idx)
-
+            print("Hinglish Normalized String:", normalized_string)
 
             # finding phoneme and leventise distance for unchanged word
-            
             for i in range(len(not_changed_idx)):
                 try:
-                    if not_changed_idx[i]==0:
-                        eng_phoneme_correction=[]
+                    if not_changed_idx[i] == 0:
+                        eng_phoneme_correction = []
                         for j in english_vocab:
-                            # print(normalized_string2[i],j)
                             try:
-                                phoneme=rs.distance(normalized_string2[i],j)
+                                phoneme = rs.distance(normalized_string2[i], j)
                             except:
                                 pass
-                            if phoneme<=1:
+                            if phoneme <= 1:
                                 eng_phoneme_correction.append(j)
-                        eng_lev_correction=[]
+                        eng_lev_correction = []
                         for k in eng_phoneme_correction:
-                            dist=lev(normalized_string2[i],k)
-                            if dist <=2:
+                            dist = lev(normalized_string2[i], k)
+                            if dist <= 2:
                                 eng_lev_correction.append(k)
-        #                 print(eng_phoneme_correction)
-        #                 print(eng_lev_correction)
-
-
-                        hing_phoneme_correction=[]
-                        for j in hinglish_vocab:
-                            try:
-                                phoneme=rs.distance(normalized_string2[i],j)
-                            except:
-                                pass
-                            if phoneme<=1:
-                                hing_phoneme_correction.append(j)
-                        hing_lev_correction=[]
-                        for k in hing_phoneme_correction:
-                            dist=lev(normalized_string2[i],k)
-                            if dist <=2:
-                                hing_lev_correction.append(k)
-        #                 print(hing_phoneme_correction)
-        #                 print(hing_lev_correction)
 
                         eng_lev_correction.extend(hing_lev_correction)
-                        new_correction=eng_lev_correction
-                        eng_lev_correction=[]
-                        # hing_lev_correction=[]
-        #                 print(eng_lev_correction)
-                        
+                        new_correction = eng_lev_correction
+                        eng_lev_correction = []
                         for l in new_correction:
-                            dist=lev(normalized_string2[i],l)
+                            dist = lev(normalized_string2[i], l)
                             eng_lev_correction.append(dist)
-                        min_val=min(eng_lev_correction)
-                        min_idx=eng_lev_correction.index(min_val)
-
-                        
-                        suggestion=dictn.suggest(new_correction[min_idx])
-                        suggestion_lit=[]
-                        for t in suggestion:
-                            dist=lev(new_correction[min_idx],t)
-                            suggestion_lit.append(dist)
-                        min_suggestion_val=min(suggestion_lit)
-                        min_suggestion_idx=suggestion_lit.index(min_suggestion_val)
-        #                 print("Suggestions : ",min_suggestion_val)
-        #                 print(suggestion[min_suggestion_idx])
+                        min_val = min(eng_lev_correction)
+                        min_idx = eng_lev_correction.index(min_val)
 
-
-
-                        normalized_string2[i]=suggestion[min_suggestion_idx]
+                        suggestion = closest_match(new_correction[min_idx], english_vocab.keys())
+                        normalized_string2[i] = suggestion
                 except:
                     pass
-            normalized_string=normalized_string2
-            normalized_string_final=normalized_string2
-            print("Phoneme levenshtein Distionary suggestion Normalized String : ",normalized_string_final)
+
+            normalized_string_final = normalized_string2
+            print("Phoneme levenshtein Distionary suggestion Normalized String:", normalized_string_final)
+
             # sentence tagging
-            classifier=joblib.load(r"./classifer.joblib")
-            classify=[]
+            classifier = joblib.load(r"./classifer.joblib")
+            classify = []
             for i in normalized_string:
-                test_classify=classifier(i)
+                test_classify = classifier(i)
                 classify.append(test_classify[0].get("label"))
 
-        #     print(normalized_string)
-        #     print(classify)
-
             for i in range(len(classify)):
-                if classify[i]=='en':
+                if classify[i] == 'en':
                     try:
-                        normalized_string[i]=translator.translate(normalized_string[i] ,src='en',dest='hi').text
+                        normalized_string[i] = translator.translate(normalized_string[i], src='en', dest='hi').text
                     except:
-                        normalized_string[i]="delete"
-            print("English -> Hindi Translated String : ",normalized_string)
-
+                        normalized_string[i] = "delete"
+            print("English -> Hindi Translated String:", normalized_string)
 
-            conversion_list=[]
+            conversion_list = [trn.transform(i) for i in normalized_string]
+            print("Hinglish -> Hindi Transliterated String:", conversion_list)
 
-            for i in tqdm(normalized_string):
-                conversion_list.append(trn.transform(i))
-
-            print("Hinglish -> Hindi Transliterated String : ",conversion_list)
-            conversion_list=normalized_string
-            string=""
-            sentence=[]
-            for i in conversion_list:
-                string=i+' '+string
-            sentence.append(string)
-            translated=[]
-            for i in tqdm(sentence):
+            sentence = [" ".join(conversion_list)]
+            translated = []
+            for i in sentence:
                 try:
-                    translated_text = translator.translate(i ,src='hi',dest='en')
+                    translated_text = translator.translate(i, src='hi', dest='en')
                     translated.append(translated_text.text)
                 except:
                     translated.append("delete")
-            print("Hindi -> English Translated String : ",translated)
+            print("Hindi -> English Translated String:", translated)
             total_translated.append(translated[0])
 
-            total_translated=pd.DataFrame(total_translated)
-
-
-
-
-        st.write("English Normalized String:", normalized_string)
-        st.write("Hinglish Normalized String:", normalized_string)
-        st.write("Phoneme Levenshtein Dictionary Suggestion Normalized String:", normalized_string_final)
-        st.write("English -> Hindi Translated String:", normalized_string)
-        st.write("Hinglish -> Hindi Transliterated String:", conversion_list)
-        st.write("Hindi -> English Translated String:", translated)
+            st.write("English Normalized String:", normalized_string)
+            st.write("Hinglish Normalized String:", normalized_string)
+            st.write("Phoneme Levenshtein Dictionary Suggestion Normalized String:", normalized_string_final)
+            st.write("English -> Hindi Translated String:", normalized_string)
+            st.write("Hinglish -> Hindi Transliterated String:", conversion_list)
+            st.write("Hindi -> English Translated String:", translated)
 
 if __name__ == '__main__':
-    main()
+    main()