Update app.py

app.py

@@ -182,16 +182,25 @@ if uploaded_file is not None:
             sentences = sentence_tokenize(text)
             for sent in sentences:
               x = mask_generation(sent)
-            
-              sent_n_q_c=[]  
-              sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
-              for i in sent_n:
-                  for j in i.split(" "):
-                      sent_n_q_c+=j.split(',')
-              x_q = x.lower().replace('.',' ').split(' ')    
-              e=[]
-              for i in x_q:
-                 e+=i.split(',')                     
+              sent_out = re.sub("\n","|",sent)
+              sent_out = re.sub(".","|",sent_out)
+              sent_out = re.sub(",","|",sent_out)
+              sent_out = re.sub(" ","|",sent_out)
+              sent_n_q_c=sent_out.split("|")
+              # sent_n_q_c=[]  
+              # sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
+              # for i in sent_n:
+              #     for j in i.split(" "):
+              #         sent_n_q_c+=j.split(',')
+              # x_q = x.lower().replace('.',' ').split(' ')    
+              # e=[]
+              # for i in x_q:
+              #    e+=i.split(',')           
+              sent_out = re.sub("\n","|",x)
+              sent_out = re.sub(".","|",sent_out)
+              sent_out = re.sub(",","|",sent_out)
+              sent_out = re.sub(" ","|",sent_out)   
+              e = sent_out.split("|")       
               t5_words=set(sent_n_q_c).difference(set(e))      
               entities,words_out = extract_entities(sent)
               # print("\nwords_out:",words_out)