Languages increased from 21 to 36

app.py

@@ -14,23 +14,38 @@ MAX_MODELS_LOADED = 5
 MODEL_IDLE_TIMEOUT = 15 * 60   # 15 minutes
 CLEANUP_INTERVAL = 15 * 60      # check every 15 minutes
 
-# Trimmed list: 21 target languages
+# Complete list: 36 target languages
 MODELS = {
     "Assamese": "prachuryyaIITG/CLASSER_Assamese_MuRIL",
     "Bengali": "prachuryyaIITG/MultiCoNER2_Bengali_XLM",
+    "Bhojpuri": "prachuryyaIITG/FiNE-MiBBiC_Bhojpuri_MuRIL",
+    "Bishnupriya": "prachuryyaIITG/FiNE-MiBBiC_Bishnupriya_MuRIL",
     "Bodo": "prachuryyaIITG/CLASSER_Bodo_MuRIL",
+    "Chhattisgarhi": "prachuryyaIITG/FiNE-MiBBiC_Chhattisgarhi_MuRIL",
     "Chinese": "prachuryyaIITG/MultiCoNER2_Chinese_XLM",
+    "Dogri": "prachuryyaIITG/SampurNER_Dogri_IndicBERTv2",
     "English": "prachuryyaIITG/MultiCoNER2_English_XLM",
     "Farsi": "prachuryyaIITG/MultiCoNER2_Farsi_XLM",
     "French": "prachuryyaIITG/MultiCoNER2_French_XLM",
     "German": "prachuryyaIITG/MultiCoNER2_German_XLM",
+    "Gujarati": "prachuryyaIITG/SampurNER_Gujarati_IndicBERTv2",
     "Hindi": "prachuryyaIITG/MultiCoNER2_Hindi_XLM",
     "Italian": "prachuryyaIITG/MultiCoNER2_Italian_XLM",
+    "Kannada": "prachuryyaIITG/SampurNER_Kannada_IndicBERTv2",
+    "Kashmiri": "prachuryyaIITG/SampurNER_Kashmiri_IndicBERTv2",
+    "Konkani": "prachuryyaIITG/SampurNER_Konkani_IndicBERTv2",
+    "Maithili": "prachuryyaIITG/SampurNER_Maithili_IndicBERTv2",
+    "Malayalam": "prachuryyaIITG/SampurNER_Malayalam_IndicBERTv2",
+    "Manipuri": "prachuryyaIITG/FiNERVINER_Manipuri_IndicBERTv2",
     "Marathi": "prachuryyaIITG/CLASSER_Marathi_MuRIL",
     "Mizo": "prachuryyaIITG/FiNERVINER_Mizo_XLM",
     "Nepali": "prachuryyaIITG/CLASSER_Nepali_MuRIL",
+    "Odia": "prachuryyaIITG/SampurNER_Odia_IndicBERTv2",
     "Portuguese": "prachuryyaIITG/MultiCoNER2_Portuguese_XLM",
+    "Punjabi": "prachuryyaIITG/SampurNER_Punjabi_IndicBERTv2",
     "Sanskrit": "prachuryyaIITG/CLASSER_Sanskrit_MuRIL",
+    "Santali": "prachuryyaIITG/SampurNER_Santali_IndicBERTv2",
+    "Sindhi": "prachuryyaIITG/SampurNER_Sindhi_IndicBERTv2",
     "Spanish": "prachuryyaIITG/MultiCoNER2_Spanish_XLM",
     "Swedish": "prachuryyaIITG/MultiCoNER2_Swedish_XLM",
     "Tamil": "prachuryyaIITG/APTFiNER_Tamil_MuRIL",
@@ -39,28 +54,61 @@ MODELS = {
     "Urdu": "prachuryyaIITG/Urdu_CLASSER_XLM",
 }
 
-# Fine-grained tag mappings into coarse categories
+# Unified mapping for MultiCoNER2, CLASSER, and FewNERD / SampurNER taxonomies
 TAG_TO_COARSE = {
-    # Person
+    # --- PERSON ---
+    # MultiCoNER2 / CLASSER / FiNERVINER / APTFiNER
     "Scientist": "PERSON", "Artist": "PERSON", "Athlete": "PERSON",
     "Politician": "PERSON", "Cleric": "PERSON", "SportsManager": "PERSON",
     "OtherPER": "PERSON", "PER": "PERSON", "Person": "PERSON",
-    
-    # Location
+    # SampurNER (person-*)
+    "Actor": "PERSON", "Artist/Author": "PERSON", "Director": "PERSON",
+    "Scholar": "PERSON", "Soldier": "PERSON", "person-actor": "PERSON",
+    "person-artist/author": "PERSON", "person-athlete": "PERSON",
+    "person-director": "PERSON", "person-other": "PERSON",
+    "person-politician": "PERSON", "person-scholar": "PERSON",
+    "person-soldier": "PERSON",
+
+    # --- LOCATION & FACILITIES ---
+    # MultiCoNER2 / CLASSER
     "Facility": "LOCATION", "OtherLOC": "LOCATION",
     "HumanSettlement": "LOCATION", "Station": "LOCATION",
     "LOC": "LOCATION", "Location": "LOCATION",
-    
-    # Organization / Group
+    # FewNERD / SampurNER (location-* & building-*)
+    "GPE": "LOCATION", "Body of Water": "LOCATION", "Island": "LOCATION",
+    "Mountain": "LOCATION", "Park": "LOCATION", "Road/Transit": "LOCATION",
+    "Airport": "LOCATION", "Hospital": "LOCATION", "Hotel": "LOCATION",
+    "Library": "LOCATION", "Restaurant": "LOCATION", "Sports Facility": "LOCATION",
+    "Theater": "LOCATION", "location-GPE": "LOCATION", "location-bodiesofwater": "LOCATION",
+    "location-island": "LOCATION", "location-mountain": "LOCATION", "location-other": "LOCATION",
+    "location-park": "LOCATION", "location-road/railway/highway/transit": "LOCATION",
+    "building-airport": "LOCATION", "building-hospital": "LOCATION", "building-hotel": "LOCATION",
+    "building-library": "LOCATION", "building-other": "LOCATION", "building-restaurant": "LOCATION",
+    "building-sportsfacility": "LOCATION", "building-theater": "LOCATION",
+
+    # --- ORGANIZATION ---
+    # MultiCoNER2 / CLASSER
     "MusicalGRP": "ORGANIZATION", "PublicCORP": "ORGANIZATION",
     "PrivateCORP": "ORGANIZATION", "AerospaceManufacturer": "ORGANIZATION",
     "SportsGRP": "ORGANIZATION", "CarManufacturer": "ORGANIZATION",
     "ORG": "ORGANIZATION", "GRP": "ORGANIZATION", "Organization": "ORGANIZATION",
-    
-    # Medical
+    # FewNERD / SampurNER (organization-*)
+    "Company": "ORGANIZATION", "Education": "ORGANIZATION", "Government": "ORGANIZATION",
+    "Media": "ORGANIZATION", "Political Party": "ORGANIZATION", "Religion": "ORGANIZATION",
+    "Sports League": "ORGANIZATION", "Show Organization": "ORGANIZATION",
+    "organization-company": "ORGANIZATION", "organization-education": "ORGANIZATION",
+    "organization-government/governmentagency": "ORGANIZATION", "organization-media/newspaper": "ORGANIZATION",
+    "organization-other": "ORGANIZATION", "organization-politicalparty": "ORGANIZATION",
+    "organization-religion": "ORGANIZATION", "organization-showorganization": "ORGANIZATION",
+    "organization-sportsleague": "ORGANIZATION", "organization-sportsteam": "ORGANIZATION",
+
+    # --- MEDICAL ---
+    # MultiCoNER2 / CLASSER
     "Medication/Vaccine": "MEDICAL", "MedicalProcedure": "MEDICAL",
     "AnatomicalStructure": "MEDICAL", "Symptom": "MEDICAL",
-    "Disease": "MEDICAL", "MED": "MEDICAL", "Medical": "MEDICAL"
+    "Disease": "MEDICAL", "MED": "MEDICAL", "Medical": "MEDICAL",
+    # FewNERD / SampurNER (misc-*)
+    "misc-disease": "MEDICAL", "misc-medical": "MEDICAL", "misc-biology": "MEDICAL"
 }
 
 # Cache and locking
@@ -142,12 +190,9 @@ def cpu_fallback_infer(text, language):
     return ner(text, stride=64)
 
 # --- PUNCTUATION & SPAN CLEANUP HELPER ---
-PUNCT_PATTERN = r'^[\s\.,!?;:"\'\(\)\[\]\{\}।॥،؟’”…—]+|[\s\.,!?;:"\'\(\)\[\]\{\}।॥،؟’”…—]+$'
-
 def clean_span_boundaries(text, start, end):
     """
     Trims leading and trailing punctuation/whitespace from span character offsets.
-    Prevents punctuation attached to words (e.g. 'Real Madrid.') from being included in the entity.
     """
     val = text[start:end]
     
@@ -175,7 +220,7 @@ def extract_regex_spans(text):
         spans.append({'start': m.start(), 'end': m.end(), 'category': 'EMAIL', 'text': m.group()})
         
     # Script-Aware Phone Numbers
-    digits = r'0-9\u0966-\u096F\u09E6-\u09EF\u0660-\u0669\u06F0-\u06F9\u0B66-\u0B6F\u0BE6-\u0BEF\u0C66-\u0C6F\uFF10-\uFF19'
+    digits = r'0-9\u0966-\u096F\u09E6-\u09EF\u0660-\u0669\u06F0-\u06F9\u0B66-\u0B6F\u0BE6-\u0BEF\u0C66-\u0C6F\u0A66-\u0A6F\u0AE6-\u0AEF\u0CDE-\u0CEF\u0D66-\u0D6F\uFF10-\uFF19'
     phone_pattern = rf'(?:\+?[' + digits + r']{1,3}[-.\s]?)?\(?[' + digits + r']{2,4}\)?[-.\s]?[' + digits + r']{3,4}[-.\s]?[' + digits + r']{3,4}\b'
     for m in re.finditer(phone_pattern, text):
         if len(re.sub(rf'[^{digits}]', '', m.group())) >= 7:
@@ -208,10 +253,19 @@ def process_pii_anonymization(text, language):
     ner_spans = []
     for res in raw_ner_results:
         entity_type = res.get('entity_group', res.get('entity', ''))
-        entity_clean = entity_type.replace("B-", "").replace("I-", "").split("_")[0]
+        # Normalize entity tag string (handles B-, I-, sub-types)
+        entity_clean = entity_type.replace("B-", "").replace("I-", "")
         
+        # Check both full clean tag and prefix split
+        matched_cat = None
         if entity_clean in TAG_TO_COARSE:
-            coarse_cat = TAG_TO_COARSE[entity_clean]
+            matched_cat = TAG_TO_COARSE[entity_clean]
+        elif entity_clean.split("_")[0] in TAG_TO_COARSE:
+            matched_cat = TAG_TO_COARSE[entity_clean.split("_")[0]]
+        elif entity_clean.split("-")[0] in TAG_TO_COARSE:
+            matched_cat = TAG_TO_COARSE[entity_clean.split("-")[0]]
+            
+        if matched_cat:
             start_pos = int(res['start'])
             end_pos = int(res['end'])
             
@@ -222,7 +276,7 @@ def process_pii_anonymization(text, language):
                 ner_spans.append({
                     'start': start_pos,
                     'end': end_pos,
-                    'category': coarse_cat,
+                    'category': matched_cat,
                     'text': clean_val
                 })
 
@@ -290,7 +344,7 @@ body, .gradio-container {
 with gr.Blocks(theme=gr.themes.Soft(), css=custom_css) as demo:
 
     gr.Markdown("# Multilingual PII Anonymizer & Synthetic Pseudonymizer")
-    gr.Markdown("Anonymize PII (**PERSON**, **LOCATION**, **ORGANIZATION**, **MEDICAL**, Emails, Phones, IPs, Credit Cards) across **21 languages** into synthetic placeholders.")
+    gr.Markdown("Anonymize sensitive PII (**PERSON**, **LOCATION**, **ORGANIZATION**, **MEDICAL**, Emails, Phones, IPs, Credit Cards) across **36 languages** into synthetic placeholders.")
 
     with gr.Row():
         lang_dropdown = gr.Dropdown(
@@ -341,6 +395,8 @@ with gr.Blocks(theme=gr.themes.Soft(), css=custom_css) as demo:
             ["Albert Einstein wurde in Ulm geboren. Er litt an Diabetes.", "German"],
             ["مرزا غالب دہلی میں رہتے تھے۔", "Urdu"],
             ["Victor Hugo est né à Besançon. Appelez le +33-1-4268-5300.", "French"],
+            ["ਰਵਿੰਦਰਨਾਥ ਟੈਗੋਰ ਕਲਕੱਤੇ ਵਿੱਚ ਰਹਿੰਦੇ ਸਨ।", "Punjabi"],
+            ["ಶಿವರಾಮ ಕಾರಂತರು ಪುತ್ತೂರಿನಲ್ಲಿ ಜನಿಸಿದರು.", "Kannada"],
         ],
         inputs=[input_text, lang_dropdown],
         outputs=[sanitized_output, mapping_json],