Delete app.py

app.py

@@ -1,972 +0,0 @@
-import gradio as gr
-import re
-import os
-import requests
-import time
-import logging
-from pathlib import Path
-
-# تنظیم logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-def auto_setup_models_for_hf():
-    """راه‌اندازی خودکار مدل‌ها برای هاگینگ فیس (بدون ورودی کاربر)"""
-    models_dir = Path("./models")
-    models_dir.mkdir(exist_ok=True)
-    
-    required_models = {
-        'bert-fa-ner': 'HooshvareLab/bert-fa-zwnj-base-ner',
-        'bert-base-NER': 'dslim/bert-base-NER',
-    }
-    
-    try:
-        # بررسی نصب transformers
-        from transformers import AutoTokenizer, AutoModelForTokenClassification
-        logger.info("✅ Transformers library available")
-        
-        for model_name, hf_repo in required_models.items():
-            model_path = models_dir / model_name
-            
-            # اگر مدل وجود دارد، رد شو
-            if model_path.exists() and list(model_path.glob("*.json")):
-                logger.info(f"✅ {model_name} already exists")
-                continue
-            
-            try:
-                logger.info(f"📥 Auto-downloading {model_name} from {hf_repo}...")
-                model_path.mkdir(exist_ok=True)
-                
-                tokenizer = AutoTokenizer.from_pretrained(hf_repo)
-                model = AutoModelForTokenClassification.from_pretrained(hf_repo)
-                
-                tokenizer.save_pretrained(model_path)
-                model.save_pretrained(model_path)
-                
-                logger.info(f"✅ {model_name} downloaded successfully")
-                del tokenizer, model  # آزادسازی حافظه
-                
-            except Exception as e:
-                logger.error(f"❌ Failed to download {model_name}: {e}")
-                # پاک کردن فایل‌های ناقص
-                if model_path.exists():
-                    import shutil
-                    shutil.rmtree(model_path)
-        
-        return True
-        
-    except ImportError:
-        logger.error("❌ transformers library not available")
-        return False
-    except Exception as e:
-        logger.error(f"❌ Auto-setup failed: {e}")
-        return False
-
-class ComprehensiveBilingualDataAnonymizer:
-    def __init__(self):
-        self.mapping_table = {}
-        # دسته‌بندی‌های الگوها برای UI
-        self.pattern_categories = {
-            'personal_identity': {
-                'name_fa': 'اطلاعات شخصی و هویتی',
-                'name_en': 'Personal & Identity Information',
-                'patterns': ['PERSON', 'MIXED_NAMES', 'ID_NUMBER', 'ENGLISH_TITLES'],
-                'icon': '👤'
-            },
-            'financial': {
-                'name_fa': 'اطلاعات مالی',
-                'name_en': 'Financial Information',
-                'patterns': ['AMOUNT', 'INTERNATIONAL_CURRENCIES', 'ACCOUNT', 'FINANCIAL_TERMS', 'STOCK_SYMBOL'],
-                'icon': '💰'
-            },
-            'temporal': {
-                'name_fa': 'اطلاعات زمانی',
-                'name_en': 'Temporal Information',
-                'patterns': ['DATE', 'ADVANCED_DATE_FORMATS', 'TIME_RANGES'],
-                'icon': '📅'
-            },
-            'location': {
-                'name_fa': 'اطلاعات مکانی',
-                'name_en': 'Location Information',
-                'patterns': ['LOCATION', 'COMPLEX_ADDRESSES'],
-                'icon': '📍'
-            },
-            'technical': {
-                'name_fa': 'اطلاعات فنی و تکنولوژیکی',
-                'name_en': 'Technical & Technological',
-                'patterns': ['TECHNICAL_CODES', 'NETWORK_ADDRESSES', 'TECHNICAL_UNITS', 'ACRONYMS_ABBREVIATIONS'],
-                'icon': '⚙️'
-            },
-            'business': {
-                'name_fa': 'اطلاعات کسب‌وکار',
-                'name_en': 'Business Information',
-                'patterns': ['COMPANY', 'BUSINESS_TERMS', 'PRODUCT', 'PETROCHEMICAL'],
-                'icon': '🏢'
-            },
-            'quantity': {
-                'name_fa': 'اطلاعات کمیت و واحد',
-                'name_en': 'Quantity & Unit Information',
-                'patterns': ['PERCENTAGE', 'VOLUME', 'RATIOS'],
-                'icon': '📊'
-            },
-            'communication': {
-                'name_fa': 'اطلاعات ارتباطی',
-                'name_en': 'Communication Information',
-                'patterns': ['PHONE', 'EMAIL'],
-                'icon': '📞'
-            }
-        }
-        
-        # counters
-        self.counters = {
-            'PERSON': 0, 'MIXED_NAMES': 0, 'ID_NUMBER': 0, 'ENGLISH_TITLES': 0,
-            'AMOUNT': 0, 'INTERNATIONAL_CURRENCIES': 0, 'ACCOUNT': 0, 
-            'FINANCIAL_TERMS': 0, 'STOCK_SYMBOL': 0,
-            'DATE': 0, 'ADVANCED_DATE_FORMATS': 0, 'TIME_RANGES': 0,
-            'LOCATION': 0, 'COMPLEX_ADDRESSES': 0,
-            'TECHNICAL_CODES': 0, 'NETWORK_ADDRESSES': 0, 'TECHNICAL_UNITS': 0,
-            'ACRONYMS_ABBREVIATIONS': 0,
-            'COMPANY': 0, 'BUSINESS_TERMS': 0, 'PRODUCT': 0, 'PETROCHEMICAL': 0,
-            'PERCENTAGE': 0, 'VOLUME': 0, 'RATIOS': 0,
-            'PHONE': 0, 'EMAIL': 0
-        }
-        
-        self.api_key = os.getenv("OPENAI_API_KEY", "")
-        self.models_base_path = "./models"
-        self.models_loaded = False
-        self.model_status = {}
-        self.load_local_ner_models()
-    
-    def get_category_choices(self, language='fa'):
-        """دریافت لیست دسته‌بندی‌ها برای چک‌باکس"""
-        choices = []
-        for cat_key, cat_info in self.pattern_categories.items():
-            name = cat_info['name_fa'] if language == 'fa' else cat_info['name_en']
-            icon = cat_info['icon']
-            choices.append(f"{icon} {name}")
-        return choices
-    
-    def get_selected_patterns(self, selected_categories, language='fa'):
-        """تبدیل دسته‌بندی‌های انتخاب شده به لیست الگوها"""
-        selected_patterns = []
-        
-        for cat_key, cat_info in self.pattern_categories.items():
-            name = cat_info['name_fa'] if language == 'fa' else cat_info['name_en']
-            icon = cat_info['icon']
-            category_display = f"{icon} {name}"
-            
-            if category_display in selected_categories:
-                selected_patterns.extend(cat_info['patterns'])
-        
-        return selected_patterns
-    
-    def load_local_ner_models(self):
-        """لود مدل‌های NER محلی با مدیریت خطا برای هاگینگ فیس"""
-        logger.info("📄 Loading local NER models (HuggingFace compatible)...")
-        
-        if not Path(self.models_base_path).exists():
-            try:
-                Path(self.models_base_path).mkdir(exist_ok=True)
-                logger.info(f"📁 Created models directory: {self.models_base_path}")
-            except Exception as e:
-                logger.error(f"❌ Failed to create models directory: {e}")
-                self.model_status['directory'] = f"❌ Cannot create models directory: {e}"
-                self.models_loaded = False
-                return
-        
-        try:
-            # بررسی نصب transformers
-            try:
-                import torch
-                from transformers import pipeline, AutoTokenizer, AutoModelForTokenClassification
-                self.model_status['transformers'] = "✅ Transformers library available"
-                logger.info("✅ Transformers library available")
-            except ImportError as e:
-                self.model_status['transformers'] = f"❌ Transformers not installed: {str(e)}"
-                self.models_loaded = False
-                logger.error(f"❌ Transformers not available: {e}")
-                return
-            
-            # تلاش برای لود مدل فارسی
-            persian_model_path = Path(self.models_base_path) / "bert-fa-ner"
-            if persian_model_path.exists() and list(persian_model_path.glob("*.json")):
-                try:
-                    self.persian_ner = pipeline("ner", 
-                                              model=str(persian_model_path), 
-                                              tokenizer=str(persian_model_path),
-                                              device=-1)  # استفاده از CPU
-                    self.model_status['persian'] = f"✅ Persian NER loaded: {persian_model_path}"
-                    logger.info("✅ Persian NER model loaded successfully")
-                except Exception as e:
-                    self.persian_ner = None
-                    self.model_status['persian'] = f"❌ Persian loading error: {str(e)[:100]}"
-                    logger.error(f"❌ Persian model loading error: {e}")
-            else:
-                self.persian_ner = None
-                self.model_status['persian'] = f"❌ Persian model not found: {persian_model_path}"
-                logger.warning(f"Persian model not found at {persian_model_path}")
-            
-            # تلاش برای لود مدل انگلیسی
-            english_model_path = Path(self.models_base_path) / "bert-base-NER"
-            if english_model_path.exists() and list(english_model_path.glob("*.json")):
-                try:
-                    self.english_ner = pipeline("ner", 
-                                               model=str(english_model_path), 
-                                               tokenizer=str(english_model_path),
-                                               device=-1)  # استفاده از CPU
-                    self.model_status['english'] = f"✅ English NER loaded: {english_model_path}"
-                    logger.info("✅ English NER model loaded successfully")
-                except Exception as e:
-                    self.english_ner = None
-                    self.model_status['english'] = f"❌ English loading error: {str(e)[:100]}"
-                    logger.error(f"❌ English model loading error: {e}")
-            else:
-                self.english_ner = None
-                self.model_status['english'] = f"❌ English model not found: {english_model_path}"
-                logger.warning(f"English model not found at {english_model_path}")
-            
-            # تعیین وضعیت نهایی
-            loaded_models = sum(1 for status in self.model_status.values() if status.startswith("✅"))
-            self.models_loaded = loaded_models > 0
-            
-            if loaded_models == 0:
-                self.model_status['fallback'] = "⚠️ Using regex-only mode (no local models found)"
-                logger.info("⚠️ No NER models loaded - using regex-only mode")
-            else:
-                logger.info(f"✅ Loaded {loaded_models} NER models successfully")
-                
-        except Exception as e:
-            self.models_loaded = False
-            self.model_status['critical'] = f"❌ Critical error: {str(e)[:100]}..."
-            logger.error(f"❌ Critical error in NER loading: {e}")
-
-    def detect_language(self, text):
-        """تشخیص زبان متن"""
-        if not text:
-            return 'fa'
-        
-        persian_chars = len(re.findall(r'[\u0600-\u06FF]', text))
-        english_chars = len(re.findall(r'[a-zA-Z]', text))
-        total = persian_chars + english_chars
-        
-        if total == 0:
-            return 'fa'
-        
-        if persian_chars / total > 0.6:
-            return 'fa'
-        elif english_chars / total > 0.6:
-            return 'en'
-        else:
-            return 'mixed'
-
-    def extract_entities_with_ner(self, text, lang='fa'):
-        """استخراج entities با مدل‌های NER محلی"""
-        entities = []
-        
-        if not self.models_loaded:
-            logger.info("ℹ️ Local NER models not available - using regex only")
-            return entities
-        
-        try:
-            # مدل فارسی
-            if lang in ['fa', 'mixed'] and hasattr(self, 'persian_ner') and self.persian_ner:
-                try:
-                    persian_results = self.persian_ner(text)
-                    for entity in persian_results:
-                        if isinstance(entity, dict):
-                            # پردازش نتایج بر اساس ساختار خروجی
-                            entity_text = entity.get('word', '').strip()
-                            entity_label = entity.get('entity_group', entity.get('entity', ''))
-                            entity_score = entity.get('score', 0)
-                            
-                            if entity_text and len(entity_text) > 1 and entity_score > 0.5:
-                                entities.append({
-                                    'text': entity_text,
-                                    'label': entity_label,
-                                    'start': entity.get('start', 0),
-                                    'end': entity.get('end', 0),
-                                    'confidence': entity_score,
-                                    'source': 'local_persian_ner'
-                                })
-                    logger.info(f"Persian NER found {len(persian_results)} entities")
-                except Exception as e:
-                    logger.error(f"Persian NER extraction error: {e}")
-            
-            # مدل انگلیسی
-            if lang in ['en', 'mixed'] and hasattr(self, 'english_ner') and self.english_ner:
-                try:
-                    english_results = self.english_ner(text)
-                    for entity in english_results:
-                        if isinstance(entity, dict):
-                            entity_text = entity.get('word', '').strip()
-                            entity_label = entity.get('entity_group', entity.get('entity', ''))
-                            entity_score = entity.get('score', 0)
-                            
-                            if entity_text and len(entity_text) > 1 and entity_score > 0.5:
-                                entities.append({
-                                    'text': entity_text,
-                                    'label': entity_label,
-                                    'start': entity.get('start', 0),
-                                    'end': entity.get('end', 0),
-                                    'confidence': entity_score,
-                                    'source': 'local_english_ner'
-                                })
-                    logger.info(f"English NER found {len(english_results)} entities")
-                except Exception as e:
-                    logger.error(f"English NER extraction error: {e}")
-                    
-        except Exception as e:
-            logger.error(f"NER extraction general error: {e}")
-        
-        # حذف تکراری‌ها
-        unique_entities = []
-        seen = set()
-        for entity in entities:
-            key = (entity['text'].lower(), entity['start'], entity['end'])
-            if key not in seen:
-                seen.add(key)
-                unique_entities.append(entity)
-        
-        logger.info(f"Total unique entities found by local models: {len(unique_entities)}")
-        return unique_entities
-
-    def map_ner_to_categories(self, ner_label, source=''):
-        """نگاشت برچسب‌های NER به دسته‌های سیستم"""
-        mapping = {
-            'PER': 'PERSON', 'PERSON': 'PERSON',
-            'ORG': 'COMPANY', 'ORGANIZATION': 'COMPANY',
-            'LOC': 'LOCATION', 'LOCATION': 'LOCATION',
-            'MISC': 'BUSINESS_TERMS', 'MISCELLANEOUS': 'BUSINESS_TERMS',
-            'B-PER': 'PERSON', 'I-PER': 'PERSON',
-            'B-ORG': 'COMPANY', 'I-ORG': 'COMPANY',
-            'B-LOC': 'LOCATION', 'I-LOC': 'LOCATION',
-            'B-MISC': 'BUSINESS_TERMS', 'I-MISC': 'BUSINESS_TERMS',
-            'MONEY': 'AMOUNT', 'PERCENT': 'PERCENTAGE',
-            'DATE': 'DATE', 'TIME': 'DATE'
-        }
-        return mapping.get(ner_label.upper(), 'BUSINESS_TERMS')
-
-    def get_comprehensive_patterns(self):
-        """الگوهای جامع ناشناس‌سازی - نسخه فشرده برای هاگینگ فیس"""
-        return {
-            'PERSON': [
-                r'آقای\s+([آ-یa-zA-Z]+(?:\s+[آ-یa-zA-Z]+)*)',
-                r'خانم\s+([آ-یa-zA-Z]+(?:\s+[آ-یa-zA-Z]+)*)',
-                r'مهندس\s+([آ-یa-zA-Z]+(?:\s+[آ-یa-zA-Z]+)*)',
-                r'دکتر\s+([آ-یa-zA-Z]+(?:\s+[آ-یa-zA-Z]+)*)',
-                r'Mr\.\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)',
-                r'Ms\.\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)',
-                r'Dr\.\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)'
-            ],
-            
-            'MIXED_NAMES': [
-                r'([آ-یa-zA-Z]{2,}\s+[آ-یa-zA-Z]{2,})',
-                r'([A-Z][a-z]+-[A-Z][a-z]+)'
-            ],
-            
-            'ID_NUMBER': [
-                r'IR[۰-۹0-9]{24}',
-                r'(?:کد[\s]*)?(?:ملی[\s:]*)?[۰-۹0-9]{10}',
-                r'(?:کارت[\s:]*)?(?:[۰-۹0-9]{4}[-\s]?){3}[۰-۹0-9]{4}',
-                r'SSN[\s:]*[0-9]{3}-[0-9]{2}-[0-9]{4}'
-            ],
-            
-            'ENGLISH_TITLES': [
-                r'business\s+partner',
-                r'team\s+lead',
-                r'senior\s+architect',
-                r'facility\s+manager'
-            ],
-            
-            'AMOUNT': [
-                r'\d+(?:,\d{3})*\s*(?:میلیون|میلیارد|هزار)\s*تومان',
-                r'مبلغ\s+\d+(?:,\d{3})*\s*(?:میلیون|میلیارد|هزار)?\s*تومان',
-                r'\$\d+(?:,\d{3})*(?:\.\d+)?\s*(?:million|billion|thousand|M|B|K)?',
-                r'€\d+(?:,\d{3})*(?:\.\d+)?'
-            ],
-            
-            'INTERNATIONAL_CURRENCIES': [
-                r'\d+(?:,\d{3})*\s+euro',
-                r'\d+(?:,\d{3})*\s+AED',
-                r'£\d+(?:,\d{3})*(?:\.\d+)?'
-            ],
-            
-            'ACCOUNT': [
-                r'(?:شماره[\s]*)?(?:حساب[\s]*)?(?:[۰-۹0-9]{1,3}[-\s]?)*[۰-۹0-9]{8,20}',
-                r'Account[\s]*(?:Number[\s:]*)?(?:[0-9]{1,3}[-\s]?)*[0-9]{8,20}'
-            ],
-            
-            'FINANCIAL_TERMS': [
-                r'فروش\s+(?:ماهانه|تجمیعی|صادراتی)',
-                r'درآمد\s+شرکت',
-                r'سود\s+(?:خالص|نقدی)',
-                r'صورت‌های\s+مالی'
-            ],
-            
-            'STOCK_SYMBOL': [
-                r'نماد\s+([آ-یa-zA-Z0-9]+)',
-                r'(AAPL|GOOGL|MSFT|AMZN|TSLA|META|NVDA|SABIC)(?=\s|$|,|\.)'
-            ],
-            
-            'DATE': [
-                r'[۰-۹0-9]{4}[/-][۰-۹0-9]{1,2}[/-][۰-۹0-9]{1,2}',
-                r'(?:[۰-۹0-9]{1,2})\s*(?:فروردین|اردیبهشت|خرداد|تیر|مرداد|شهریور|مهر|آبان|آذر|دی|بهمن|اسفند)\s*(?:[۰-۹0-9]{4})',
-                r'(?:[0-9]{1,2})\s*(?:January|February|March|April|May|June|July|August|September|October|November|December)\s*(?:[0-9]{4})'
-            ],
-            
-            'ADVANCED_DATE_FORMATS': [
-                r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z',
-                r'(?:PST|EST|GMT|UTC)(?:[+-]\d{1,2}:\d{2})?'
-            ],
-            
-            'TIME_RANGES': [
-                r'\d{2}:\d{2}-\d{2}:\d{2}',
-                r'\d{1,2}:\d{2}\s+(?:AM|PM)\s+(?:PST|EST|GMT|UTC)'
-            ],
-            
-            'LOCATION': [
-                r'(تهران|اصفهان|ماهشهر|عسلویه|بندرعباس|اهواز|شیراز|مشهد|تبریز|کرج)',
-                r'(London|Paris|Tokyo|New\s+York|Dubai|Singapore)'
-            ],
-            
-            'COMPLEX_ADDRESSES': [
-                r'کیلومتر\s+\d+\s+جاده\s+[آ-ی\s]+-[آ-ی\s]+',
-                r'Building-[A-Z],?\s+Floor-\d+,?\s+Unit-[A-Z0-9]+'
-            ],
-            
-            'TECHNICAL_CODES': [
-                r'SN-\d{4}-[A-Z]{3}-\d{4}',
-                r'REF-[A-Z]{3}-\d{4}-\d{3}',
-                r'HVAC-\d{7}'
-            ],
-            
-            'NETWORK_ADDRESSES': [
-                r'\b(?:\d{1,3}\.){3}\d{1,3}\b',
-                r'[A-F0-9]{2}:[A-F0-9]{2}:[A-F0-9]{2}:[A-F0-9]{2}:[A-F0-9]{2}:[A-F0-9]{2}'
-            ],
-            
-            'TECHNICAL_UNITS': [
-                r'\d+(?:\.\d+)?\s*MW',
-                r'\d+(?:\.\d+)?\s*kWh?',
-                r'\d+(?:,\d{3})*\s*cubic\s+meters'
-            ],
-            
-            'ACRONYMS_ABBREVIATIONS': [
-                r'\b(?:HVAC|IT|HSE|BOQ|LC|COB)\b',
-                r'\b(?:LLC|Corp|Inc|Ltd)\b'
-            ],
-            
-            'COMPANY': [
-                r'شرکت(?=\s+در|\s+که|\s+با|\s+را|\s+به|\s+طی)',
-                r'(بانک\s+[آ-یa-zA-Z\s]+)',
-                r'([A-Z][a-zA-Z\s]+(?:Inc|Corp|Corporation|Company|Ltd|Limited|LLC))'
-            ],
-            
-            'BUSINESS_TERMS': [
-                r'تحلیل\s+عملکرد',
-                r'گزارش\s+(?:فعالیت|عملکرد)\s+ماهانه',
-                r'تولید\s+پایدار'
-            ],
-            
-            'PRODUCT': [
-                r'\b(?:VCM|PVC|PE|PP|PS|ABS|SAN|PC|PMMA|PET|PBT|PA|POM|TPU)\b',
-                r'پلی\s*(?:اتیلن|پروپیلن|استایرن)'
-            ],
-            
-            'PETROCHEMICAL': [
-                r'\b(?:LDPE|HDPE|LLDPE|PP|PS|EPS|ABS)\b'
-            ],
-            
-            'PERCENTAGE': [
-                r'\d+(?:\.\d+)?\s*درصد(?:\s+افزایش|\s+رشد|\s+کاهش)?',
-                r'\d+(?:\.\d+)?\s*%',
-                r'\d+(?:\.\d+)?\%\s*(?:increase|decrease|growth)'
-            ],
-            
-            'VOLUME': [
-                r'\d+(?:,\d{3})*\s*تن',
-                r'\d+(?:,\d{3})*\s*(?:tons|kg|liters|barrels)'
-            ],
-            
-            'RATIOS': [
-                r'نسبت\s+(?:فروش|تولید)\s+به\s+[آ-ی\s]+',
-                r'برابر\s+با\s+\d+(?:\.\d+)?'
-            ],
-            
-            'PHONE': [
-                r'(?:تلفن[\s:]*)?(?:شماره[\s:]*)?(?:0)?(?:[۰-۹0-9]{2,3}[-\s]?)?[۰-۹0-9]{7,8}',
-                r'(?:موبایل[\s:]*)?(?:شماره[\s:]*)?(?:0)?9[۰-۹0-9]{9}',
-                r'\+[0-9]{1,3}-[0-9]{3}-[0-9]{3}-[0-9]{4}'
-            ],
-            
-            'EMAIL': [
-                r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}',
-                r'ایمیل[\s:]*[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
-            ]
-        }
-
-    def anonymize_text(self, original_text, lang='fa', selected_categories=None):
-        """گام 1: ناشناس‌سازی متن با الگوهای انتخاب شده"""
-        try:
-            if not original_text or not original_text.strip():
-                return "❌ Please enter input text!" if lang == 'en' else "❌ لطفاً متن ورودی را وارد کنید!"
-            
-            # ریست متغیرها
-            self.mapping_table = {}
-            self.counters = {key: 0 for key in self.counters.keys()}
-            
-            anonymized = original_text
-            found_entities = set()
-            
-            # تشخیص زبان
-            detected_lang = self.detect_language(original_text)
-            logger.info(f"Detected language: {detected_lang}")
-            
-            # مرحله 1: استخراج با Local NER
-            if self.models_loaded:
-                logger.info("🤖 Running local NER extraction...")
-                ner_entities = self.extract_entities_with_ner(original_text, detected_lang)
-                
-                for entity in ner_entities:
-                    if (entity['text'] not in found_entities and 
-                        len(entity['text'].strip()) > 1 and 
-                        entity['confidence'] > 0.5):
-                        
-                        category = self.map_ner_to_categories(entity['label'], entity['source'])
-                        
-                        if entity['text'] not in self.mapping_table:
-                            self.counters[category] += 1
-                            code = f"{category}_{self.counters[category]:03d}_LOCAL_NER"
-                            self.mapping_table[entity['text']] = code
-                            found_entities.add(entity['text'])
-                            logger.info(f"Local NER: {entity['text']} -> {code}")
-            else:
-                logger.info("ℹ️ Using regex-only mode")
-            
-            # مرحله 2: الگوهای Regex انتخاب شده
-            all_patterns = self.get_comprehensive_patterns()
-            
-            # فیلتر کردن الگوها بر اساس انتخاب کاربر
-            if selected_categories:
-                selected_pattern_types = self.get_selected_patterns(selected_categories, lang)
-                patterns = {k: v for k, v in all_patterns.items() if k in selected_pattern_types}
-                logger.info(f"📋 Using selected pattern categories: {len(patterns)} types")
-            else:
-                patterns = all_patterns
-                logger.info("📋 Using all available pattern categories")
-            
-            # پردازش patterns
-            logger.info("🔍 Running selective regex extraction...")
-            
-            processed_entities = set()
-            
-            # اولویت‌بندی دسته‌ها
-            priority_order = [
-                'ID_NUMBER', 'EMAIL', 'PHONE', 'ACCOUNT', 'TECHNICAL_CODES',
-                'NETWORK_ADDRESSES', 'INTERNATIONAL_CURRENCIES', 'AMOUNT',
-                'TECHNICAL_UNITS', 'ACRONYMS_ABBREVIATIONS', 'ADVANCED_DATE_FORMATS',
-                'TIME_RANGES', 'COMPLEX_ADDRESSES', 'MIXED_NAMES', 'ENGLISH_TITLES',
-                'STOCK_SYMBOL', 'COMPANY', 'PERSON', 'PERCENTAGE', 'VOLUME',
-                'RATIOS', 'LOCATION', 'DATE', 'FINANCIAL_TERMS', 'BUSINESS_TERMS',
-                'PRODUCT', 'PETROCHEMICAL'
-            ]
-            
-            for category in priority_order:
-                if category in patterns:
-                    pattern_list = patterns[category]
-                    for pattern in pattern_list:
-                        try:
-                            matches = re.finditer(pattern, original_text, re.IGNORECASE | re.MULTILINE)
-                            for match in matches:
-                                if match.groups():
-                                    item = match.group(1).strip()
-                                    full_match = match.group(0).strip()
-                                else:
-                                    item = match.group(0).strip()
-                                    full_match = item
-                                
-                                # بررسی تداخل
-                                overlaps = False
-                                match_start, match_end = match.span()
-                                
-                                for proc_start, proc_end in processed_entities:
-                                    if not (match_end <= proc_start or match_start >= proc_end):
-                                        overlaps = True
-                                        break
-                                
-                                if (not overlaps and
-                                    full_match not in found_entities and 
-                                    full_match not in self.mapping_table and 
-                                    len(full_match) >= 2):
-                                    
-                                    self.counters[category] += 1
-                                    code = f"{category}_{self.counters[category]:03d}_REGEX"
-                                    self.mapping_table[full_match] = code
-                                    found_entities.add(full_match)
-                                    processed_entities.add((match_start, match_end))
-                                    logger.info(f"Regex ({category}): {full_match} -> {code}")
-                        except re.error as e:
-                            logger.error(f"Regex error in pattern {pattern}: {e}")
-                            continue
-            
-            # جایگزینی در متن
-            sorted_items = sorted(self.mapping_table.items(), key=lambda x: len(x[0]), reverse=True)
-            for original_item, code in sorted_items:
-                anonymized = anonymized.replace(original_item, code)
-            
-            logger.info(f"✅ Selective anonymization completed. Found {len(self.mapping_table)} entities.")
-            return anonymized
-            
-        except Exception as e:
-            logger.error(f"Anonymization error: {e}")
-            return f"❌ Error in anonymization: {str(e)}" if lang == 'en' else f"❌ خطا در ناشناس‌سازی: {str(e)}"
-
-    def send_to_chatgpt(self, anonymized_text, lang='fa'):
-        """گام 2: ارسال به ChatGPT"""
-        try:
-            if not anonymized_text or not anonymized_text.strip():
-                return "❌ Anonymized text is empty!" if lang == 'en' else "❌ متن ناشناس‌شده خالی است!"
-            
-            if not self.api_key:
-                return "❌ API Key not configured! Please set OPENAI_API_KEY environment variable." if lang == 'en' else "❌ کلید API تنظیم نشده است!"
-            
-            system_msg = "You are a professional analyst. Answer questions accurately." if lang == 'en' else "شما یک تحلیلگر حرفه‌ای هستید. به سوالات با دقت پاسخ دهید."
-            
-            headers = {
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json"
-            }
-            
-            data = {
-                "model": "gpt-4o-mini",
-                "messages": [
-                    {"role": "system", "content": system_msg},
-                    {"role": "user", "content": anonymized_text}
-                ],
-                "max_tokens": 2000,
-                "temperature": 0.7
-            }
-            
-            response = requests.post(
-                "https://api.openai.com/v1/chat/completions",
-                headers=headers,
-                json=data,
-                timeout=30
-            )
-            
-            if response.status_code == 200:
-                result = response.json()
-                return result['choices'][0]['message']['content']
-            else:
-                error_data = response.json() if response.content else {}
-                error_message = error_data.get('error', {}).get('message', response.text)
-                return f"❌ API Error: {error_message}"
-                
-        except Exception as e:
-            return f"❌ Error connecting to ChatGPT: {str(e)}" if lang == 'en' else f"❌ خطا در ارتباط با ChatGPT: {str(e)}"
-
-    def deanonymize_response(self, gpt_response, lang='fa'):
-        """گام 3: بازگردانی"""
-        try:
-            if not gpt_response or not gpt_response.strip():
-                return "❌ ChatGPT response is empty!" if lang == 'en' else "❌ پاسخ ChatGPT خالی است!"
-            
-            if not self.mapping_table:
-                return "❌ Mapping table is empty!" if lang == 'en' else "❌ جدول نگاشت خالی است!"
-            
-            final_result = gpt_response
-            reverse_mapping = {code: original for original, code in self.mapping_table.items()}
-            
-            sorted_codes = sorted(reverse_mapping.items(), key=lambda x: len(x[0]), reverse=True)
-            for code, original in sorted_codes:
-                final_result = final_result.replace(code, original)
-            
-            return final_result
-            
-        except Exception as e:
-            return f"❌ Deanonymization error: {str(e)}" if lang == 'en' else f"❌ خطا در بازگردانی: {str(e)}"
-
-    def get_model_status(self):
-        """وضعیت مدل‌های محلی"""
-        status = "🤖 **HuggingFace Compatible Anonymization System Status:**\n\n"
-        
-        if hasattr(self, 'model_status') and self.model_status:
-            for model_type, model_status in self.model_status.items():
-                status += f"• **{model_type.title()}**: {model_status}\n"
-        
-        loaded_count = sum(1 for status in getattr(self, 'model_status', {}).values() 
-                          if status.startswith("✅"))
-        status += f"\n📊 **Summary**: {loaded_count}/2 local models loaded"
-        status += f"\n🔍 **Models Path**: {self.models_base_path}"
-        status += f"\n🔧 **Environment**: HuggingFace Spaces Compatible"
-        
-        status += f"\n\n🎯 **Available Pattern Categories:**"
-        for cat_key, cat_info in self.pattern_categories.items():
-            icon = cat_info['icon']
-            name_fa = cat_info['name_fa']
-            pattern_count = len(cat_info['patterns'])
-            status += f"\n   {icon} {name_fa}: {pattern_count} patterns"
-        
-        status += f"\n\n✨ **System Features:**"
-        status += f"\n   🎯 User-controlled category selection"
-        status += f"\n   🛡️ Flexible sensitive data protection"
-        status += f"\n   📊 Efficient targeted processing"
-        status += f"\n   ⚡ HuggingFace Spaces optimized"
-        
-        return status
-
-# ایجاد instance
-anonymizer = ComprehensiveBilingualDataAnonymizer()
-
-def process_all_steps(input_text, language, selected_categories):
-    """پردازش خودکار تمام مراحل"""
-    lang = 'en' if language == 'English' else 'fa'
-    
-    if not input_text.strip():
-        error_msg = "❌ Please enter input text!" if lang == 'en' else "❌ لطفاً متن ورودی را وارد کنید!"
-        return error_msg, "", "", ""
-    
-    try:
-        start_time = time.time()
-        
-        anonymized_text = anonymizer.anonymize_text(input_text, lang, selected_categories)
-        if anonymized_text.startswith("❌"):
-            return anonymized_text, "", "", ""
-        
-        gpt_response = anonymizer.send_to_chatgpt(anonymized_text, lang)
-        if gpt_response.startswith("❌"):
-            entities_found = len(anonymizer.mapping_table)
-            ner_count = sum(1 for code in anonymizer.mapping_table.values() if '_NER' in code)
-            regex_count = sum(1 for code in anonymizer.mapping_table.values() if '_REGEX' in code)
-            
-            selected_count = len(selected_categories) if selected_categories else 0
-            
-            method = "Selective Local NER + Regex" if anonymizer.models_loaded else "Selective Regex Only"
-            success_msg = (f"✅ Selective anonymization completed with {method}!\n"
-                          f"��� Selected categories: {selected_count} | 🤖 NER: {ner_count} | 🔍 Regex: {regex_count}\n"
-                          f"📊 Total protected entities: {entities_found}")
-            return success_msg, anonymized_text, gpt_response, ""
-        
-        final_result = anonymizer.deanonymize_response(gpt_response, lang)
-        
-        total_time = time.time() - start_time
-        entities_found = len(anonymizer.mapping_table)
-        ner_count = sum(1 for code in anonymizer.mapping_table.values() if '_NER' in code)
-        regex_count = sum(1 for code in anonymizer.mapping_table.values() if '_REGEX' in code)
-        
-        selected_count = len(selected_categories) if selected_categories else 8
-        
-        method = "Selective Local NER + Regex" if anonymizer.models_loaded else "Selective Regex Only"
-        success_msg = (f"🎉 Complete selective anonymization & restoration successful!\n"
-                      f"🔧 Method: {method} | 📋 Categories: {selected_count}/8\n"
-                      f"📊 Total: {entities_found} entities | 🤖 NER: {ner_count} | 🔍 Regex: {regex_count}\n"
-                      f"⏱️ Time: {total_time:.2f}s | 🎯 HuggingFace optimized")
-        
-        return success_msg, anonymized_text, gpt_response, final_result
-        
-    except Exception as e:
-        error_msg = f"❌ Processing error: {str(e)}" if lang == 'en' else f"❌ خطا در پردازش: {str(e)}"
-        return error_msg, "", "", ""
-
-def get_mapping_table(language):
-    """نمایش جدول نگاشت"""
-    lang = 'en' if language == 'English' else 'fa'
-    
-    if not anonymizer.mapping_table:
-        return "❌ Mapping table is empty!" if lang == 'en' else "❌ جدول نگاشت خالی است!"
-    
-    result = "📋 **Selective Mapping Table:**\n\n"
-    
-    # نمایش آمار کلی
-    ner_count = sum(1 for code in anonymizer.mapping_table.values() if '_NER' in code)
-    regex_count = sum(1 for code in anonymizer.mapping_table.values() if '_REGEX' in code)
-    
-    result += f"📊 **Statistics**: {len(anonymizer.mapping_table)} total entities\n"
-    result += f"🤖 **NER Detected**: {ner_count} entities\n"
-    result += f"🔍 **Regex Detected**: {regex_count} entities\n\n"
-    
-    # نمایش نمونه‌ها
-    if ner_count > 0:
-        result += "🤖 **NER Results (Sample)**:\n"
-        ner_items = [(k, v) for k, v in anonymizer.mapping_table.items() if '_NER' in v]
-        for original, code in ner_items[:3]:
-            result += f"   • `{original}` → `{code}`\n"
-        result += "\n"
-    
-    if regex_count > 0:
-        result += "🔍 **Regex Results (Sample)**:\n"
-        regex_items = [(k, v) for k, v in anonymizer.mapping_table.items() if '_REGEX' in v]
-        for original, code in regex_items[:5]:
-            result += f"   • `{original}` → `{code}`\n"
-        result += "\n"
-    
-    result += "✨ **System**: HuggingFace Spaces compatible with selective processing!"
-    
-    return result
-
-def clear_all():
-    """پاک کردن همه"""
-    anonymizer.mapping_table = {}
-    anonymizer.counters = {key: 0 for key in anonymizer.counters.keys()}
-    return "", "", "", "", ""
-
-# CSS ساده برای هاگینگ فیس
-custom_css = """
-.gradio-container {
-    font-family: 'Segoe UI', Arial, sans-serif !important;
-    max-width: 1200px !important;
-    margin: 0 auto !important;
-}
-
-.rtl {
-    direction: rtl !important;
-    text-align: right !important;
-}
-
-.gradio-textbox {
-    border-radius: 8px !important;
-    min-height: 150px !important;
-}
-
-.gradio-button {
-    border-radius: 8px !important;
-    font-weight: bold !important;
-}
-"""
-
-# رابط کاربری Gradio ساده برای هاگینگ فیس
-with gr.Blocks(title="Selective Anonymization System", theme=gr.themes.Soft(), css=custom_css) as app:
-    
-    gr.HTML("<h1 style='text-align: center; color: #2563eb;'>📊 سیستم ناشناس‌سازی انتخابی</h1>")
-    
-    with gr.Row():
-        language_selector = gr.Radio(
-            choices=["فارسی", "English"],
-            value="فارسی",
-            label="Language / زبان"
-        )
-    
-    with gr.Row():
-        with gr.Column(scale=2):
-            gr.HTML("<h3>🎯 انتخاب دسته‌بندی‌های مورد نظر:</h3>")
-            pattern_categories = gr.CheckboxGroup(
-                choices=anonymizer.get_category_choices('fa'),
-                value=anonymizer.get_category_choices('fa'),
-                label="دسته‌بندی‌های الگو"
-            )
-        
-        with gr.Column(scale=3):
-            input_text = gr.Textbox(
-                lines=8,
-                placeholder="متن خود را اینجا وارد کنید...",
-                label="متن ورودی",
-                rtl=True
-            )
-    
-    with gr.Row():
-        process_btn = gr.Button("🚀 پردازش با دسته‌بندی‌های انتخاب شده", variant="primary", size="lg")
-        clear_btn = gr.Button("🗑️ پاک کردن", variant="secondary")
-    
-    status = gr.Textbox(
-        label="وضعیت",
-        lines=3,
-        interactive=False,
-        rtl=True
-    )
-    
-    with gr.Row():
-        with gr.Column():
-            gr.HTML("<h3>🎭 متن ناشناس‌شده</h3>")
-            anonymized_output = gr.Textbox(
-                lines=6,
-                interactive=False,
-                rtl=True
-            )
-        
-        with gr.Column():
-            gr.HTML("<h3>🤖 پاسخ ChatGPT</h3>")
-            gpt_output = gr.Textbox(
-                lines=6,
-                interactive=False,
-                rtl=True
-            )
-    
-    gr.HTML("<h3>✅ پاسخ نهایی بازگردانده شده</h3>")
-    final_output = gr.Textbox(
-        lines=6,
-        interactive=False,
-        rtl=True
-    )
-    
-    with gr.Row():
-        mapping_btn = gr.Button("📋 نمایش جدول نگاشت")
-        status_btn = gr.Button("📊 وضعیت سیستم")
-    
-    with gr.Row():
-        mapping_output = gr.Textbox(
-            lines=10,
-            label="جدول نگاشت",
-            interactive=False,
-            visible=False,
-            rtl=True
-        )
-        
-        system_status_output = gr.Textbox(
-            lines=15,
-            label="وضعیت سیستم",
-            interactive=False,
-            visible=False,
-            rtl=True
-        )
-
-    # Event handlers
-    process_btn.click(
-        fn=process_all_steps,
-        inputs=[input_text, language_selector, pattern_categories],
-        outputs=[status, anonymized_output, gpt_output, final_output]
-    )
-
-    clear_btn.click(
-        fn=clear_all,
-        outputs=[input_text, anonymized_output, gpt_output, final_output, status]
-    )
-
-    mapping_btn.click(
-        fn=get_mapping_table,
-        inputs=[language_selector],
-        outputs=[mapping_output]
-    )
-
-    mapping_btn.click(
-        fn=lambda: gr.update(visible=True),
-        outputs=[mapping_output]
-    )
-    
-    status_btn.click(
-        fn=lambda: anonymizer.get_model_status(),
-        outputs=[system_status_output]
-    )
-    
-    status_btn.click(
-        fn=lambda: gr.update(visible=True),
-        outputs=[system_status_output]
-    )
-
-# تلاش برای دانلود خودکار مدل‌ها در startup
-if __name__ == "__main__":
-    logger.info("🚀 Starting HuggingFace compatible anonymization system...")
-    
-    # تلاش برای دانلود خودکار مدل‌ها
-    try:
-        auto_setup_models_for_hf()
-    except Exception as e:
-        logger.warning(f"⚠️ Auto-setup issue: {e}")
-    
-    logger.info("✅ System ready for HuggingFace Spaces!")
-    
-    app.launch(
-        share=False,  # در هاگینگ فیس share=False بهتر است
-        server_name="0.0.0.0",
-        server_port=7860,
-        show_error=True
-    )