updated email extractor

app.py

@@ -135,6 +135,31 @@ def email_extractor(email_uploaded):
     
     return email_body, character_cnt, url_cnt
 
+def email_extractor_general(email_uploaded):
+    parse = parse_email(email_uploaded)
+    email_text = ''.join(parse).strip()
+
+    # get rid of non-text elements
+    email_text = email_text.replace('\n', '')
+    email_text = email_text.replace('\t', '')
+    email_text = email_text.replace('\r', '')
+    email_text = email_text.replace('</b>', '')
+    email_text = email_text.replace('<b>', '')
+    email_text = email_text.replace('\xa0', '')
+    
+    # find length of URLs if any
+    extractor = URLExtract()
+    urls = extractor.find_urls(email_text)
+    url_cnt = len(urls)
+    
+    # remove URLs and get character count
+    body = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', email_text)    
+    sep = '©'
+    body = body.split(sep, 1)[0]
+    character_cnt = sum(not chr.isspace() for chr in body)
+    
+    return email_text, character_cnt, url_cnt
+
 
 # extract email body from parse email
 def email_body_extractor(email_data):
@@ -363,7 +388,7 @@ if st.session_state.get('button') == True:
         #uploaded_file = FileChooser(uploaded_file)
         #bytes_data = uploaded_file.getvalue()
 
-        email_body, character_cnt, url_cnt = email_extractor(uploaded_file)
+        email_body, character_cnt, url_cnt = email_extractor_general(uploaded_file)
 
         # Start the prediction
         # Need to solve X test issue