Update cultural_tokenizer.py

cultural_tokenizer.py

@@ -1,12 +1,10 @@
 from transformers import M2M100Tokenizer
 
 class CulturalTokenizer(M2M100Tokenizer):
-    """Custom tokenizer with UNESCO language support"""
+    """Tokenizer with UNESCO language enhancements"""
     
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        
-        # Add special tokens
         self.add_special_tokens({
             'additional_special_tokens': [
                 '<CULTURAL_CONTEXT>',
@@ -14,17 +12,14 @@ class CulturalTokenizer(M2M100Tokenizer):
                 '<VULNERABLE>'
             ]
         })
-        
-        # Language vitality mapping
-        self.vitality_tags = {
-            'ay': '<VULNERABLE>',
-            'chr': '<ENDANGERED>',
-            'qu': '<VULNERABLE>'
-        }
     
-    def prepare_seq2seq_batch(self, *args, **kwargs):
-        """Add vitality tags to source text"""
-        src_lang = kwargs.get('src_lang')
-        if src_lang in self.vitality_tags:
-            kwargs['src_text'] = f"{self.vitality_tags[src_lang]} {kwargs['src_text']}"
-        return super().prepare_seq2seq_batch(*args, **kwargs)
+    def prepare_seq2seq_batch(self, src_text, **kwargs):
+        """Add cultural metadata to endangered languages"""
+        tgt_lang = kwargs.get('tgt_lang')
+        
+        if tgt_lang in ['ay', 'qu']:  # Vulnerable languages
+            src_text = f"<VULNERABLE> {src_text}"
+        elif tgt_lang in ['chr']:  # Endangered languages
+            src_text = f"<ENDANGERED> {src_text}"
+        
+        return super().prepare_seq2seq_batch(src_text, **kwargs)