Update app.py

app.py

@@ -6,6 +6,7 @@ import fitz  # PyMuPDF
 from docx import Document
 import re
 import nltk
+from nltk import word_tokenize
 from presidio_analyzer import AnalyzerEngine, PatternRecognizer, RecognizerResult, Pattern
 nltk.download('punkt')
 
@@ -182,26 +183,30 @@ if uploaded_file is not None:
             sentences = sentence_tokenize(text)
             for sent in sentences:
               x = mask_generation(sent)
+              sent_words = word_tokenize(sent.lower())
+              t5_words = word_tokenize(x.lower())
+
+              t5_words=list(set(sent_words).difference(set(t5_words)))
               # sent_out = re.sub("\n","|",sent)
               # sent_out = re.sub(".","|",sent_out)
               # sent_out = re.sub(",","|",sent_out)
               # sent_out = re.sub(" ","|",sent_out)
               # sent_n_q_c=sent_out.split("|")
-              sent_n_q_c=[]  
-              sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
-              for i in sent_n:
-                  for j in i.split(" "):
-                      sent_n_q_c+=j.split(',')
-              x_q = x.lower().replace('.',' ').split(' ')    
-              e=[]
-              for i in x_q:
-                 e+=i.split(',')           
+              # sent_n_q_c=[]  
+              # sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
+              # for i in sent_n:
+              #     for j in i.split(" "):
+              #         sent_n_q_c+=j.split(',')
+              # x_q = x.lower().replace('.',' ').split(' ')    
+              # e=[]
+              # for i in x_q:
+              #    e+=i.split(',')           
               # sent_out = re.sub("\n","|",x)
               # sent_out = re.sub(".","|",sent_out)
               # sent_out = re.sub(",","|",sent_out)
               # sent_out = re.sub(" ","|",sent_out)   
               # e = sent_out.split("|")       
-              t5_words=list(set(sent_n_q_c).difference(set(e)))
+              # t5_words=list(set(sent_n_q_c).difference(set(e)))
               entities,words_out = extract_entities(sent)
               words_out+=t5_words
               # print("\nwords_out:",words_out)