Update app.py

app.py

@@ -1,8 +1,8 @@
 
 import re
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
 import threading
+from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
 
 TAG_PREFIX = "N"
 
@@ -27,6 +27,7 @@ OCCUPATIONAL_TITLES = ['대표', '이사', '전무', '상무', '부장', '차장
     '교감', '부교장', '조교수', '교수', '연구원', '강사', '박사', '석사', '학사',
     '의사', '간호사', '간병인', '보호자', '피해자', '당사자', '대상자', '주민', '어르신', '기사님']
 COMMON_SUFFIXES = FAMILY_TITLES + ACADEMIC_TITLES + OCCUPATIONAL_TITLES
+COMMON_JOSA = ['이', '가', '은', '는', '을', '를', '께서', '에게', '에서']
 
 model_name = "Leo97/KoELECTRA-small-v3-modu-ner"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -45,149 +46,95 @@ def to_chosung(text):
             result += ch
     return result
 
-def expand_name_with_prefix_suffix(text, base_names):
-    detected = set()
-    for name in base_names:
-        pattern1 = re.compile(rf'([가-힣]){re.escape(name)}(학생|선생|씨|님)?(이|가|은|는|을|를|께서|에게|에서)?')
-        for m in pattern1.finditer(text):
-            detected.add(m.group(0))
-        pattern2 = re.compile(rf'{re.escape(name)}(씨|님)?(이|가|은|는|을|를|께서|에게|에서)?')
-        for m in pattern2.finditer(text):
-            detected.add(m.group(0))
-    return list(detected)
-
 def extract_names(text):
-    try:
-        results = ner_pipeline(text)
-    except Exception as e:
-        print("NER 오류 발생:", e)
-        return []
+    results = ner_pipeline(text)
     names = []
-    base_names = set()
     for entity in results:
         if entity.get("entity_group") == "PS":
             name = entity["word"].replace("##", "").strip()
             if len(name) >= 2 and name not in NAME_ENTITY_EXCEPTIONS:
                 names.append(name)
-                base_names.add(name)
-    extended = expand_name_with_prefix_suffix(text, base_names)
-    for name in extended:
-        if name not in names:
-            names.append(name)
-    return names
-
-def refactored_mask_names(original_text, names, start_counter=100):
-    korean_josa = ['이가','를','은','는','을','도','만','과','와','에게','에서','으로','까지','조차','마저','이며','이다','이나','이나마','밖에','이든','이라도','이','가','의']
-    masked = original_text
+    return list(set(names))
+
+def apply_name_tags(text, names, start=100):
     mapping = {}
-    counter = start_counter
-    used_names = set()
+    tagged = text
+    counter = start
     for name in names:
-        for josa in korean_josa:
-            full = name + josa
-            pattern = rf'(?<![\w가-힣]){re.escape(full)}(?![\w가-힣])'
-            if re.search(pattern, masked):
-                tag = f"{TAG_PREFIX}{counter:03d}"
-                mapping[tag] = name
-                masked = re.sub(pattern, tag + josa, masked)
-                counter += 1
-                used_names.add(name)
-                break
-    for name in names:
-        if name in used_names:
-            continue
-        pattern = rf'(?<![\w가-힣]){re.escape(name)}(?![\w가-힣])'
-        if re.search(pattern, masked):
-            tag = f"{TAG_PREFIX}{counter:03d}"
+        tag = f"{TAG_PREFIX}{counter:03d}"
+        pattern = re.compile(rf'(?<![\w가-힣]){re.escape(name)}(?![\w가-힣])')
+        tagged, n = pattern.subn(tag, tagged)
+        if n > 0:
             mapping[tag] = name
-            masked = re.sub(pattern, tag, masked)
             counter += 1
-    return masked, mapping
-
-def final_name_remask_exact_only(text, mapping_dict):
-    for tag, name in mapping_dict.items():
-        pattern = rf'(?<![\w가-힣]){re.escape(name)}(?![\w가-힣])'
-        text = re.sub(pattern, tag, text)
-    return text
+    return tagged, mapping
 
-def expand_suffix_name_mapping(text, mapping_dict):
+def expand_from_tag_context(tagged_text, mapping):
     updated = {}
-    for tag, name in mapping_dict.items():
-        candidates = []
-        pattern1 = rf"([가-힣]){re.escape(name)}(?:{'|'.join(COMMON_SUFFIXES)})?(이|가|은|는|을|를|께서|에게|에서)?"
-        for m in re.finditer(pattern1, text):
-            candidates.append(m.group(0))
-        pattern2 = rf"{re.escape(name)}(씨|님)?(이|가|은|는|을|를|께서|에게|에서)?"
-        for m in re.finditer(pattern2, text):
-            candidates.append(m.group(0))
-        updated[tag] = max(candidates, key=len) if candidates else name
+    for tag, name in mapping.items():
+        idx = tagged_text.find(tag)
+        if idx == -1:
+            updated[tag] = name
+            continue
+        context = tagged_text[max(0, idx - 50): idx + 50]
+        pattern = re.compile(rf'([가-힣])?{re.escape(name)}({"|".join(COMMON_SUFFIXES)})?({"|".join(COMMON_JOSA)})?')
+        matches = pattern.findall(context)
+        if matches:
+            longest = max(matches, key=lambda x: len(''.join(x)))
+            updated[tag] = ''.join(longest)
+        else:
+            updated[tag] = name
     return updated
 
-def postprocess_sensitive_patterns(text):
-    text = re.sub(r"\b[\w\.-]+@", r"******@", text)
-    def mask_sequence(match):
-        parts = re.split(r'[.-]', match.group())
-        masked = [(part if i % 2 == 0 else '*' * len(part)) if part.isdigit() else part for i, part in enumerate(parts)]
-        return '.'.join(masked) if '.' in match.group() else '-'.join(masked)
-    text = re.sub(r"(?<![\\$\\\\])(?<!\d,)(?:\d{2,4}[.-]){1,3}\d{2,4}(?!\d)", mask_sequence, text)
-    text = re.sub(r"(\d{1,3})동", r"***동", text)
-    text = re.sub(r"(\d{1,4})호", r"****호", text)
-    return text
+def mask_school_names(text):
+    def replace_school(m):
+        return to_chosung(m.group(1)) + m.group(2)
+    return re.sub(r"([가-힣]{2,20})(초등학교|중학교|고등학교)", replace_school, text)
 
 def mask_department(text):
-    return re.sub(r"([가-힣]{2,20}학과)", lambda m: to_chosung(m.group(1)[:-2]) + "학과", text)
+    return re.sub(r"([가-힣]{2,20})학과", lambda m: to_chosung(m.group(1)) + "학과", text)
 
-def mask_school_names(text):
-    global school_name_candidates
-    school_name_candidates = []
-    def replacer(match):
-        name = match.group(1)
-        if 2 <= len(name) <= 20:
-            school_name_candidates.append(name)
-            return to_chosung(name) + match.group(2)
-        return match.group(0)
-    text = re.sub(r"(\b[가-힣]{2,20})(초등학교|중학교|고등학교)", replacer, text)
-    for name in school_name_candidates:
-        pattern = rf"{re.escape(name)}\s?(초등학교|중학교|고등학교)"
-        text = re.sub(pattern, to_chosung(name) + " " + r"\1", text)
+def postprocess_sensitive_patterns(text):
+    text = re.sub(r"[\w\.-]+@", "******@", text)
+    text = re.sub(r"(\d{6})[- ]?(\d{7})", "******-*******", text)
+    text = re.sub(r"(\d{3})[- ]?(\d{4})[- ]?(\d{4})", "***-****-****", text)
+    text = re.sub(r"(\d{1,3})동", "***동", text)
+    text = re.sub(r"(\d{1,4})호", "****호", text)
     return text
 
 def sanitize_sensitive_info(text, keyword_string, replace_word):
     text = postprocess_sensitive_patterns(text)
     text = mask_school_names(text)
     text = mask_department(text)
-    text = re.sub(r"(\d)학년(\s?(\d)반)?", lambda m: "*학년" + (" *반" if m.group(3) else ""), text)
+    text = re.sub(r"(\d)학년(\s?(\d)반)?", "*학년 *반", text)
     keywords = [k.strip() for k in keyword_string.split(",") if k.strip()] + list(REGEX_KEYWORDS_TO_MASK)
     for kw in keywords:
-        pattern = rf"\b{re.escape(kw)}\b"
-        text = re.sub(pattern, replace_word, text, flags=re.IGNORECASE)
-    text = re.sub(r"(\d{6})[-](\d)\d{6}", r"*******-\2*****", text)
-    text = re.sub(r"([가-힣]+(대로|로|길))\s?(\d+)(호|번길|가)?", r"\1 ***", text)
+        text = re.sub(rf"{re.escape(kw)}", replace_word, text, flags=re.IGNORECASE)
     return text
 
-def apply_masking(text, keywords, replace_word):
+def apply_masking(text, keyword_string, replace_word):
+    original = text
+    text = sanitize_sensitive_info(text, keyword_string, replace_word)
     names = extract_names(text)
-    masked, mapping = refactored_mask_names(text, names)
-    sanitized = sanitize_sensitive_info(masked, keywords, replace_word)
+    tagged, mapping = apply_name_tags(text, names)
 
-    def delayed_postprocess():
-        updated_mapping = expand_suffix_name_mapping(text, mapping)
-        final_output = final_name_remask_exact_only(text, updated_mapping)  # 원본 기준 재적용!!
+    def finalize():
+        updated_mapping = expand_from_tag_context(tagged, mapping)
         final_map = "\n".join([f"{k} → {v}" for k, v in updated_mapping.items()])
-        masked_output.update(value=final_output)
+        masked_output.update(value=tagged)
         mapping_output.update(value=final_map)
 
-    threading.Timer(0.2, delayed_postprocess).start()
-    mapping_table = "\n".join([f"{k} → {v}" for k, v in mapping.items()])
-    return sanitized, mapping_table
+    threading.Timer(0.2, finalize).start()
+    initial_map = "\n".join([f"{k} → {v}" for k, v in mapping.items()])
+    return tagged, initial_map
 
 with gr.Blocks() as demo:
-    gr.Markdown("🛡️ **민감정보 마스킹 [v3 FIXED: 태그 대응 완성]**")
-    input_text = gr.Textbox(lines=15, label="📅 원본 텍스트 입력")
+    gr.Markdown("🧠 **v4.2 ULTIMATE FULL: 태그 기반 확장 + 민감정보 마스킹 완전체**")
+    input_text = gr.Textbox(lines=15, label="📄 입력 텍스트")
     keyword_input = gr.Textbox(lines=1, label="기관 키워드 (쉼표 구분)", value="굿네이버스, 사회복지법인 굿네이버스")
     replace_input = gr.Textbox(lines=1, label="치환할 텍스트", value="우리기관")
     run_button = gr.Button("🚀 마스킹 실행")
     masked_output = gr.Textbox(lines=15, label="🔐 마스킹 결과")
-    mapping_output = gr.Textbox(lines=10, label="🏷️ 이름 태그 매핑", interactive=False)
+    mapping_output = gr.Textbox(lines=10, label="🏷️ 태그 매핑", interactive=False)
     run_button.click(fn=apply_masking, inputs=[input_text, keyword_input, replace_input], outputs=[masked_output, mapping_output])
 demo.launch()