update app

app.py

@@ -3,7 +3,7 @@ import streamlit as st
 import re 
 import logging
 from presidio_anonymizer import AnonymizerEngine
-from presidio_analyzer import AnalyzerEngine, PatternRecognizer, RecognizerResult
+from presidio_analyzer import AnalyzerEngine, PatternRecognizer, RecognizerResult, EntityRecognizer
 
 from annotated_text import annotated_text
 from flair_recognizer import FlairRecognizer
@@ -41,21 +41,21 @@ def analyze(**kwargs):
     if "entities" not in kwargs or "All" in kwargs["entities"]:
         kwargs["entities"] = None
 
-    if st.session_state.excluded_words:
-        logging.info(
-                f"type of excluded_words_recognizer: {type(st.session_state.excluded_words)}\n"
-            )
-        logging.info(
-                f"excluded words: {st.session_state.excluded_words.split(',')}\n"
-            )
-        excluded_words_recognizer = PatternRecognizer(supported_entity="MANUAL ADD",
-                                      name="Excluded words recognizer",
-                                      deny_list=st.session_state.excluded_words.split(','))
-        analyzer_engine().registry.add_recognizer(excluded_words_recognizer)
+    # if st.session_state.excluded_words:
+
+    #     deny_list = [i.strip() for i in st.session_state.excluded_words.split(',')]
+
+    #     logging.info(
+    #             f"words excluded : {deny_list}\n"
+    #         )
+
+    #     excluded_words_recognizer = PatternRecognizer(supported_entity="MANUAL ADD",
+    #                                   name="Excluded words recognizer",
+    #                                   deny_list=deny_list)
+    #     analyzer_engine().registry.add_recognizer(excluded_words_recognizer)
 
     results =  analyzer_engine().analyze(**kwargs)
-    
-    return results
+    st.session_state.analyze_results = results
 
 def annotate():
     text = st.session_state.text 
@@ -98,64 +98,76 @@ def analyze_text():
             logging.info(f"This is the text being analysed: {st.session_state.text}")
             st.session_state.text_error = ""
             st.session_state.n_requests += 1
-            analyze_results = analyze(
+            analyze(
                 text=st.session_state.text,
                 entities=st_entities,
                 language="en",
                 return_decision_process=False,
             )
 
-            # if st.session_state.excluded_words:
-            #     analyze_results = include_manual_input(analyze_results)
+            if st.session_state.excluded_words:
+                include_manual_input()
 
             if st.session_state.allowed_words:
-                analyze_results = exclude_manual_input(analyze_results)
-
-            st.session_state.analyze_results = analyze_results
+                exclude_manual_input()
            
             logging.info(
                 f"analyse results: {st.session_state.analyze_results}\n"
             )
 
 
-# def include_manual_input(analyze_results):
-#     analyze_results_extended=analyze_results
-#     logging.info(
-#                 f"analyse results before adding extra words: {analyze_results}\n"
-#             )
-#     for word in st.session_state.excluded_words:
-#         if word in st.session_state.text:
-#             r = re.compile(word)
-#             index_entries = [[m.start(),m.end()] for m in r.finditer(st.session_state.text)]
-#             for entry in index_entries:
-#                 start=entry[0]
-#                 end=entry[1]
-                
-#                 analyze_results_extended.append("type": "MANUAL ADD", "start": start, "end": end, "score": 1.0})
-#     logging.info(
-#                 f"analyse results after adding allowed words: {analyze_results_extended}\n"
-#             )
-#     logging.info(
-#                 f"type of entries in results: {type(analyze_results[0])}\n"
-#             )
-#     return analyze_results_extended
-
-
-## We might be able to create a new result from json https://github.com/microsoft/presidio/blob/07b854dd7ae247b916aef4d2adbb82f33bba7be8/presidio-analyzer/presidio_analyzer/recognizer_result.py#L72
-
-def exclude_manual_input(analyze_results):
-    analyze_results_fltered=[]
+def include_manual_input():
+    deny_list = [i.strip() for i in st.session_state.excluded_words.split(',')]
+    def _deny_list_to_regex(deny_list):
+        """
+        Convert a list of words to a matching regex.
+        To be analyzed by the analyze method as any other regex patterns.
+        :param deny_list: the list of words to detect
+        :return:the regex of the words for detection
+        """
+        # Escape deny list elements as preparation for regex
+        escaped_deny_list = [re.escape(element) for element in deny_list]
+        regex = r"(?:^|(?<=\W))(" + "|".join(escaped_deny_list) + r")(?:(?=\W)|$)"
+        return regex
+
+    deny_list_pattern = _deny_list_to_regex(deny_list)
+    matches = re.finditer(deny_list_pattern, st.session_state.text)
+    results = []
+    for match in matches:
+        start, end = match.span()
+        current_match = st.session_state.text[start:end]
+
+        # Skip empty results
+        if current_match == "":
+            continue
+            
+        pattern_result = RecognizerResult(
+                    entity_type='MANUALLY ADDED',
+                    start=start,
+                    end=end,
+                    score=1.0,
+                )
+        
+        results.append(pattern_result)
+
+    results = EntityRecognizer.remove_duplicates(results)    
+    
+    st.session_state.analyze_results = st.session_state.analyze_results.extend(results)
+
     logging.info(
-                f"analyse results before removing allowed words: {analyze_results}\n"
+                f"analyse results after adding excluded words: {results}\n"
             )
-    for token in analyze_results:
+   
+def exclude_manual_input():
+    analyze_results_fltered=[]
+
+    for token in st.session_state.analyze_results:
         if st.session_state.text[token.start:token.end] not in st.session_state.allowed_words:
             analyze_results_fltered.append(token)
     logging.info(
                 f"analyse results after removing allowed words: {analyze_results_fltered}\n"
             )
-    return analyze_results_fltered
-
+    st.session_state.analyze_results = analyze_results_fltered
 
 @st.cache(allow_output_mutation=True)
 def anonymizer_engine():
@@ -190,8 +202,7 @@ def anonymise_text():
 def clear_results():
     st.session_state.anon_results=""
     st.session_state.analyze_results=""
-    # if not st.session_state.excluded_words:
-    analyzer_engine().registry.remove_recognizer("Excluded words recognizer")
+    # analyzer_engine().registry.remove_recognizer("Excluded words recognizer")
 
 #######################################
 #### Initialize "global" variables ####