import gradio as gr import torch from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM import re # Global variables to store the models atlas_pipe = None transliteration_tokenizer = None transliteration_model = None def load_models(): """Load both Atlas-Chat and Transliteration models""" global atlas_pipe, transliteration_tokenizer, transliteration_model # Load Atlas-Chat model if atlas_pipe is None: print("🏔️ Loading Atlas-Chat-2B model...") atlas_pipe = pipeline( "text-generation", model="MBZUAI-Paris/Atlas-Chat-2B", model_kwargs={"torch_dtype": torch.bfloat16}, device="cuda" if torch.cuda.is_available() else "cpu" ) print("✅ Atlas-Chat model loaded!") # Load Transliteration model if transliteration_tokenizer is None or transliteration_model is None: print("🔄 Loading Transliteration model...") transliteration_tokenizer = AutoTokenizer.from_pretrained("atlasia/Transliteration-Moroccan-Darija") transliteration_model = AutoModelForSeq2SeqLM.from_pretrained("atlasia/Transliteration-Moroccan-Darija") print("✅ Transliteration model loaded!") return atlas_pipe, transliteration_tokenizer, transliteration_model def detect_arabizi(text): """ Detect if input text is written in Arabizi (Latin script with numbers) Returns True if Arabizi is detected """ if not text or len(text.strip()) < 2: return False # Check for Arabic script - if present, it's NOT Arabizi arabic_pattern = r'[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]' if re.search(arabic_pattern, text): return False # Arabizi indicators - numbers used as letters arabizi_numbers = ['2', '3', '7', '9', '5', '6', '8'] has_arabizi_numbers = any(num in text for num in arabizi_numbers) # Common Arabizi words and patterns arabizi_patterns = [ 'wach', 'wash', 'ach', 'achno', 'chno', 'shno', 'shkoun', 'chkoun', 'kif', 'kifash', 'ki', 'kayf', 'kien', 'kima', 'feen', 'fin', 'fen', 'fain', 'mnin', 'imta', 'meta', 'waqt', 'mata', 'emta', 'hna', 'ahna', 'ana', 'nta', 'nti', 'ntuma', 'ntouma', 'howa', 'hiya', 'huma', 'houma', 'hoa', 'hia', 'had', 'hadchi', 'hada', 'hadi', 'hadou', 'hadouk', 'bghit', 'bghiti', 'bgha', 'bghina', 'bghitiou', 'galt', 'galti', 'gal', 'galet', 'galou', 'rah', 'raha', 'rahi', 'rahom', 'rahin', 'kan', 'kanu', 'kana', 'kanet', 'kano', 'ghadi', 'ghad', 'gha', 'ghadia', 'ghadiyin', 'daba', 'dak', 'dakchi', 'dik', 'dok', 'bzf', 'bzzaf', 'bezzaf', 'bzaaaaf', 'chway', 'chwiya', 'shwiya', 'chwia', 'khoya', 'khuya', 'akhi', 'kho', 'khti', 'khtiya', 'ukhti', 'kht', 'mama', 'baba', 'lwaldin', 'lwalidin', 'salam', 'salamu aleikum', 'slm', 'yallah', 'yalla', 'hya', 'aji', 'mabghitsh', 'mabghach', 'makansh', 'machi', 'walakin', 'walaken', 'ama', 'mais', 'kayn', 'makaynsh', 'chi', 'tayi' ] text_lower = text.lower() has_arabizi_words = any(pattern in text_lower for pattern in arabizi_patterns) # Decision logic if has_arabizi_numbers and has_arabizi_words: return True if has_arabizi_numbers and len([c for c in text if c.isalpha()]) > len(text) * 0.6: return True if has_arabizi_words and len([c for c in text if c.isalpha()]) > len(text) * 0.7: return True return False def arabizi_to_arabic_ai(arabizi_text): """ Convert Arabizi text to Arabic using the specialized AI model """ try: _, tokenizer, model = load_models() # Tokenize the input text input_tokens = tokenizer(arabizi_text, return_tensors="pt", padding=True, truncation=True, max_length=512) # Perform transliteration with torch.no_grad(): output_tokens = model.generate( **input_tokens, max_length=512, num_beams=4, early_stopping=True, no_repeat_ngram_size=2 ) # Decode the output tokens arabic_text = tokenizer.decode(output_tokens[0], skip_special_tokens=True) return arabic_text.strip() except Exception as e: print(f"❌ Error in Arabizi→Arabic conversion: {e}") # Fallback to original text if conversion fails return arabizi_text def arabic_to_arabizi(arabic_text): """ Convert Arabic script to Arabizi using character mappings (Keeping this as backup since no reverse model available) """ if not arabic_text: return arabic_text # COMPREHENSIVE WORD MAPPINGS (Arabic → Arabizi) word_mappings = { # Common words first (most likely to appear) 'أنا': 'ana', 'نتا': 'nta', 'نتي': 'nti', 'هوا': 'howa', 'هيا': 'hiya', 'حنا': 'hna', 'أحنا': 'ahna', 'نتوما': 'ntuma', 'هوما': 'huma', 'شكون': 'shkoun', 'أشنو': 'achno', 'شنو': 'chno', 'واش': 'wach', 'كيفاش': 'kifash', 'كيف': 'kif', 'فين': 'feen', 'منين': 'mnin', 'إمتا': 'imta', 'متا': 'meta', 'علاش': '3lach', 'أش': 'ach', 'بغيت': 'bghit', 'بغيتي': 'bghiti', 'بغا': 'bgha', 'بغينا': 'bghina', 'كان': 'kan', 'كانا': 'kana', 'كانت': 'kanet', 'كانو': 'kanu', 'قلت': 'galt', 'قلتي': 'galti', 'قال': 'gal', 'قالت': 'galet', 'راح': 'rah', 'راها': 'raha', 'راهي': 'rahi', 'راهم': 'rahom', 'غادي': 'ghadi', 'غاد': 'ghad', 'غا': 'gha', 'هاد': 'had', 'هادا': 'hada', 'هادي': 'hadi', 'هادشي': 'hadchi', 'داك': 'dak', 'ديك': 'dik', 'داكشي': 'dakchi', 'بزاف': 'bzzaf', 'شوياة': 'chwiya', 'كولشي': 'kolchi', 'ماشي': 'machi', 'مابغيتش': 'mabghitsh', 'ماكاينش': 'makainch', 'دابا': 'daba', 'توا': 'tawa', 'غدا': 'ghda', 'ماما': 'mama', 'بابا': 'baba', 'خويا': 'khoya', 'ختي': 'khti', 'سلام': 'salam', 'يالاه': 'yallah', 'هيا': 'hya', 'المغرب': 'lmaghrib', 'مغرب': 'maghrib', 'طاجين': 'tajine', 'أتاي': 'atay', 'خوبز': 'khobz', 'كاين': 'kayn', 'ماكاينش': 'makaynsh', 'شي': 'chi', 'زوين': 'zwin', 'زوينا': 'zwina', 'مزيان': 'mzyan', 'مزيانا': 'mzyana', 'درت': 'dert', 'درتي': 'derti', 'دار': 'dar', 'درات': 'derat', 'مشيت': 'mchit', 'مشيتي': 'mchiti', 'مشا': 'mcha', 'مشات': 'mchat', 'جيت': 'jit', 'جيتي': 'jiti', 'جا': 'ja', 'جات': 'jat', 'شفت': 'cheft', 'شفتي': 'chefti', 'شاف': 'chaf', 'شافت': 'chafat', 'سمعت': 'sme3t', 'سمعتي': 'sme3ti', 'سمع': 'sma3', 'سمعات': 'sma3at', 'أكلت': 'klit', 'أكلتي': 'kliti', 'كلا': 'kla', 'كلات': 'klat', 'شربت': 'chrebt', 'شربتي': 'chrebti', 'شرب': 'chreb', 'شربات': 'chrebat', 'نعست': 'ne3st', 'نعستي': 'ne3sti', 'نعس': 'ne3s', 'نعسات': 'ne3sat', 'خرجت': 'khrjt', 'خرجتي': 'khrjti', 'خرج': 'khrj', 'خرجات': 'khrjat', 'دخلت': 'dkhlt', 'دخلتي': 'dkhlti', 'دخل': 'dkhl', 'دخلات': 'dkhlat', 'قريت': 'qrit', 'قريتي': 'qriti', 'قرا': 'qra', 'قرات': 'qrat', 'كتبت': 'ktebt', 'كتبتي': 'ktebti', 'كتب': 'kteb', 'كتبات': 'ktebat', 'لعبت': 'l3ebt', 'لعبتي': 'l3ebti', 'لعب': 'l3eb', 'لعبات': 'l3ebat', 'خدمت': 'khdmt', 'خدمتي': 'khdmti', 'خدم': 'khdm', 'خدمات': 'khdmat', 'صليت': 'sllit', 'صليتي': 'slliti', 'صلا': 'slla', 'صلات': 'sllat', 'طبخت': '6bkht', 'طبختي': '6bkhti', 'طبخ': '6bekh', 'طبخات': '6bekhat', 'واحد': 'wa7ed', 'جوج': 'joj', 'تلاتا': 'tlata', 'ربعا': 'reb3a', 'خمسا': 'khamsa', 'ستا': 'setta', 'سبعا': 'seb3a', 'تمنيا': 'tmnya', 'تسعا': 'tes3a', 'عشرا': '3echra', 'حداش': '7dach', 'طناش': '6nach', 'نهار': 'nhar', 'ليلا': 'lila', 'صباح': 'sba7', 'عشيا': '3echiya', 'أمس': 'ems', 'البارح': 'lbare7', 'غدا': 'ghda', 'بعد غدا': 'b3d ghda', 'دار': 'dar', 'بيت': 'bit', 'شارع': 'char3', 'مدينا': 'mdina', 'كرهوبا': 'karhouba', 'طوموبيل': 'tomobil', 'قطار': 'q6ar', 'باص': 'bas', 'ماكلا': 'makla', 'شراب': 'chrab', 'لما': 'lma', 'عطش': '36ch', 'جوع': 'jo3', 'شبعان': 'cheb3an', 'عيان': '3yyan', 'صحيح': 's7i7', 'مريض': 'mrid', 'دكتور': 'doktor', 'سبيطار': 'sbitar', 'دوا': 'dwa', 'فلوس': 'flous', 'درهم': 'derhem', 'ريال': 'riyal', 'اليورو': 'lyoro', 'خدما': 'khedma', 'معلم': 'mo3alim', 'طالب': 'talib', 'أستاذ': 'ostaz', 'كتاب': 'ktab', 'قلم': 'qalam', 'كاغط': 'kaghet', 'طاولا': 'tabla' } # CHARACTER MAPPINGS (Arabic → Arabizi) char_mappings = { 'ا': 'a', 'ب': 'b', 'ت': 't', 'ث': 'th', 'ج': 'j', 'ح': '7', 'خ': 'kh', 'د': 'd', 'ذ': 'dh', 'ر': 'r', 'ز': 'z', 'س': 's', 'ش': 'sh', 'ص': 's', 'ض': 'd', 'ط': '6', 'ظ': 'z', 'ع': '3', 'غ': 'gh', 'ف': 'f', 'ق': '9', 'ك': 'k', 'ل': 'l', 'م': 'm', 'ن': 'n', 'ه': 'h', 'و': 'w', 'ي': 'y', 'ء': '2', 'آ': 'aa', 'أ': 'a', 'إ': 'i', 'ة': 'a', 'ى': 'a', '؟': '?', '،': ',', '؛': ';', '：': ':', '！': '!', 'َ': 'a', 'ُ': 'o', 'ِ': 'i', 'ً': 'an', 'ٌ': 'on', 'ٍ': 'in' } result = arabic_text # Step 1: Apply word mappings for arabic_word, arabizi_word in word_mappings.items(): # Use word boundaries to avoid partial matches result = re.sub(r'\b' + re.escape(arabic_word) + r'\b', arabizi_word, result) # Step 2: Apply character mappings for arabic_char, arabizi_char in char_mappings.items(): result = result.replace(arabic_char, arabizi_char) return result.strip() def chat_with_atlas(message, history): """Generate response from Atlas-Chat model with AI-powered Arabizi conversion""" if not message.strip(): return "ahlan wa sahlan! kifash n9der n3awnek? / مرحبا! كيفاش نقدر نعاونك؟" try: # Load models atlas_model, _, _ = load_models() # Detect if input is Arabizi is_arabizi_input = detect_arabizi(message) print("\n" + "="*60) print("🔍 DEBUG LOG - FULL CONVERSION PIPELINE") print("="*60) print(f"📥 ORIGINAL INPUT: '{message}'") print(f"🤖 ARABIZI DETECTED: {is_arabizi_input}") # Prepare input for the model if is_arabizi_input: # Convert Arabizi to Arabic using AI model print(f"\n🔄 STEP 1: Converting Arabizi to Arabic...") arabic_input = arabizi_to_arabic_ai(message) print(f"✅ ARABIC CONVERSION: '{arabic_input}'") model_input = arabic_input else: # Use original input (Arabic or English) print(f"\n➡️ NO CONVERSION NEEDED - Using original input") model_input = message print(f"\n🤖 STEP 2: Sending to Atlas-Chat model...") print(f"📤 MODEL INPUT: '{model_input}'") # Generate response using Arabic input messages = [{"role": "user", "content": model_input}] outputs = atlas_model( messages, max_new_tokens=256, temperature=0.1, do_sample=True, pad_token_id=atlas_model.tokenizer.eos_token_id ) # Extract the response response = outputs[0]["generated_text"][-1]["content"].strip() print(f"✅ MODEL RESPONSE (Arabic): '{response}'") # Convert response back to Arabizi if input was Arabizi if is_arabizi_input: print(f"\n🔄 STEP 3: Converting response back to Arabizi...") arabizi_response = arabic_to_arabizi(response) print(f"✅ FINAL ARABIZI RESPONSE: '{arabizi_response}'") print("="*60) print("🎯 FINAL OUTPUT TO USER:", arabizi_response) print("="*60 + "\n") return arabizi_response else: # Return original response for Arabic/English print(f"\n➡️ NO BACK-CONVERSION NEEDED") print("="*60) print("🎯 FINAL OUTPUT TO USER:", response) print("="*60 + "\n") return response except Exception as e: print(f"\n❌ ERROR OCCURRED: {str(e)}") print("="*60 + "\n") # Return error in appropriate language if detect_arabizi(message): return f"sorry, kan chi mochkil: {str(e)}. 3awd jar'b!" else: return f"عذراً، واجهت خطأ: {str(e)}. جرب مرة أخرى! / Sorry, error occurred: {str(e)}. Try again!" # Create the Gradio interface demo = gr.ChatInterface( fn=chat_with_atlas, title="🏔️ Atlas-Chat: Advanced Moroccan Arabic AI", description=""" **مرحبا بك في أطلس شات المطور!** Welcome to Advanced Atlas-Chat! 🇲🇦 **🧠 AI-Powered Language Detection & Conversion:** - **Arabic Script (العربية)** → AI responds in Arabic - **Arabizi (3arabi bi 7oruf latin)** → AI-powered conversion → Arabizi response - **English** → AI responds in English **⚡ Professional Arabizi Conversion** - Uses specialized AI model trained on Moroccan Darija - Perfect understanding of context: "kayn chi" → "كاين شي" - Handles complex phrases accurately **جرب هذه الأسئلة / Try these questions:** """, examples=[ "شكون لي صنعك؟", "shkoun li sna3ek?", "اشنو هو الطاجين؟", "achno howa tajine?", "شنو كيتسمى المنتخب المغربي؟", "chno kaytsma lmontakhab lmaghribi?", "What is Morocco famous for?", "كيفاش نقدر نتعلم الدارجة؟", "kifash n9der nt3elem darija?", "wach kayn atay f lmaghrib?", "3lach lmaghrib zwien bzzaf?", "kifash nsali tajine?", "chno homa l2aklat lmaghribiya?", "kayn chi restaurants zwinin f casa?", "mr7ba! kif dayr?" ], cache_examples=False ) # Launch the app if __name__ == "__main__": demo.launch()