Duplicate from UBC-NLP/Simba-S

Co-authored-by: AbdelRahim Elmadany <elmadany@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,212 @@
+---
+language:
+  - am  # Amharic
+  - ar  # Arabic
+  - tw  # Asante Twi
+  - bm  # Bambara
+  - fr  # French
+  - lg  # Ganda
+  - ha  # Hausa
+  - ig  # Igbo
+  - rw  # Kinyarwanda
+  - kg  # Kongo
+  - ln  # Lingala
+  - lu  # Luba-Katanga
+  - mg  # Malagasy
+  - nso # Northern Sotho
+  - ny  # Nyanja
+  - om  # Oromo
+  - pt  # Portuguese
+  - sn  # Shona
+  - so  # Somali
+  - st  # Southern Sotho
+  - sw  # Swahili
+  - ss  # Swati
+  - ti  # Tigrinya
+  - ts  # Tsonga
+  - tn  # Tswana
+  - ak  # Twi
+  - ve  # Venda
+  - wo  # Wolof
+  - xh  # Xhosa
+  - yo  # Yoruba
+  - zu  # Zulu
+  - tzm # Tamazight
+  - sg  # Sango
+  - din # Dinka
+  - ee  # Ewe
+  - fo  # Fon
+  - luo # Luo
+  - mos # Mossi
+  - umb # Umbundu
+license: cc-by-4.0
+tags:
+  - automatic-speech-recognition
+  - audio
+  - speech
+  - african-languages
+  - multilingual
+  - simba
+  - low-resource
+  - speech-recognition
+  - asr
+datasets:
+  - UBC-NLP/SimbaBench
+metrics:
+  - wer
+  - cer
+library_name: transformers
+pipeline_tag: automatic-speech-recognition
+---
+<div align="center">
+
+<img src="https://africa.dlnlp.ai/simba/images/VoC_simba" alt="VoC Simba Models Logo">
+
+
+[![EMNLP 2025 Paper](https://img.shields.io/badge/EMNLP_2025-Paper-B31B1B?style=for-the-badge&logo=arxiv&logoColor=B31B1B&labelColor=FFCDD2)](https://aclanthology.org/2025.emnlp-main.559/)
+[![Official Website](https://img.shields.io/badge/Official-Website-2EA44F?style=for-the-badge&logo=googlechrome&logoColor=2EA44F&labelColor=C8E6C9)](https://africa.dlnlp.ai/simba/)
+[![SimbaBench](https://img.shields.io/badge/SimbaBench-Benchmark-8A2BE2?style=for-the-badge&logo=googlecharts&logoColor=8A2BE2&labelColor=E1BEE7)](https://huggingface.co/spaces/UBC-NLP/SimbaBench)
+[![GitHub Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=for-the-badge&logo=github&logoColor=181717&labelColor=E0E0E0)](https://github.com/UBC-NLP/simba)
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Models-FFD21E?style=for-the-badge&logoColor=181717&labelColor=FFF9C4)](https://huggingface.co/collections/UBC-NLP/simba-speech-series)
+[![Hugging Face Dataset](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Dataset-FFD21E?style=for-the-badge&logoColor=181717&labelColor=FFF9C4)](https://huggingface.co/datasets/UBC-NLP/SimbaBench_dataset)
+
+</div>
+
+## *Bridging the Digital Divide for African AI*
+
+**Voice of a Continent** is a comprehensive open-source ecosystem designed to bring African languages to the forefront of artificial intelligence. By providing a unified suite of benchmarking tools and state-of-the-art models, we ensure that the future of speech technology is inclusive, representative, and accessible to over a billion people.
+
+## Best-in-Class Multilingual Models
+
+Introduced in our EMNLP 2025 paper *[Voice of a Continent](https://aclanthology.org/2025.emnlp-main.559/)*, the **Simba Series** represents the current state-of-the-art for African speech AI.
+
+- **Unified Suite:** Models optimized for African languages.
+- **Superior Accuracy:** Outperforms generic multilingual models by leveraging SimbaBench's high-quality, domain-diverse datasets.
+- **Multitask Capability:** Designed for high performance in ASR (Automatic Speech Recognition) and TTS (Text-to-Speech).
+- **Inclusion-First:** Specifically built to mitigate the "digital divide" by empowering speakers of underrepresented languages.
+
+The **Simba** family consists of state-of-the-art models fine-tuned using SimbaBench. These models achieve superior performance by leveraging dataset quality, domain diversity, and language family relationships.
+
+### 🗣️✍️ Simba-ASR
+> **The New Standard for African Speech-to-Text**
+
+**🎯 Task** `Automatic Speech Recognition` — Powering high-accuracy transcription across the continent.
+
+**🌍 Language Coverage (43 African languages)**
+>  **Amharic** (`amh`), **Arabic** (`ara`), **Asante Twi** (`asanti`), **Bambara** (`bam`), **Baoulé** (`bau`), **Bemba** (`bem`), **Ewe** (`ewe`), **Fanti** (`fat`), **Fon** (`fon`), **French** (`fra`), **Ganda** (`lug`), **Hausa** (`hau`), **Igbo** (`ibo`), **Kabiye** (`kab`), **Kinyarwanda** (`kin`), **Kongo** (`kon`), **Lingala** (`lin`), **Luba-Katanga** (`lub`), **Luo** (`luo`), **Malagasy** (`mlg`), **Mossi** (`mos`), **Northern Sotho** (`nso`), **Nyanja** (`nya`), **Oromo** (`orm`), **Portuguese** (`por`), **Shona** (`sna`), **Somali** (`som`), **Southern Sotho** (`sot`), **Swahili** (`swa`), **Swati** (`ssw`), **Tigrinya** (`tir`), **Tsonga** (`tso`), **Tswana** (`tsn`), **Twi** (`twi`), **Umbundu** (`umb`), **Venda** (`ven`), **Wolof** (`wol`), **Xhosa** (`xho`), **Yoruba** (`yor`), **Zulu** (`zul`), **Tamazight** (`tzm`), **Sango** (`sag`), **Dinka** (`din`).
+
+**🏗️ Base Architectures**
+
+  -  **Simba-S** (SeamlessM4T-v2-MT) — *Top Performer*
+  - **Simba-W** (Whisper-v3-large)
+  - **Simba-X** (Wav2Vec2-XLS-R-2b)
+  - **Simba-M** (MMS-1b-all)
+  - **Simba-H** (AfriHuBERT)
+      
+🌐 Explore the Frontier
+
+| **ASR Models**   | **Architecture**  | **#Parameters** | **🤗 Hugging Face Model Card** | **Status** |
+|---------|:------------------:| :------------------:| :------------------:|:------------------:|    
+| 🔥**Simba-S**🔥|    SeamlessM4T-v2  |  2.3B | 🤗 [https://huggingface.co/UBC-NLP/Simba-S](https://huggingface.co/UBC-NLP/Simba-S) | ✅ Released |
+| 🔥**Simba-W**🔥|    Whisper         |  1.5B | 🤗 [https://huggingface.co/UBC-NLP/Simba-W](https://huggingface.co/UBC-NLP/Simba-W) | ✅ Released | 
+| 🔥**Simba-X**🔥|    Wav2Vec2        |  1B | 🤗 [https://huggingface.co/UBC-NLP/Simba-X](https://huggingface.co/UBC-NLP/Simba-X) | ✅ Released |   
+| 🔥**Simba-M**🔥|    MMS             |  1B | 🤗 [https://huggingface.co/UBC-NLP/Simba-M](https://huggingface.co/UBC-NLP/Simba-M) | ✅ Released |   
+| 🔥**Simba-H**🔥|    HuBERT          |  94M | 🤗 [https://huggingface.co/UBC-NLP/Simba-H](https://huggingface.co/UBC-NLP/Simba-H) | ✅ Released |   
+
+* **Simba-S** emerged as the best-performing ASR model overall.
+
+
+**🧩 Usage Example**
+
+You can easily run inference using the Hugging Face `transformers` library.
+
+```python
+from transformers import pipeline
+
+# Load Simba-S for ASR
+asr_pipeline = pipeline(
+    "automatic-speech-recognition",
+    model="UBC-NLP/Simba-S" #Simba mdoels `UBC-NLP/Simba-S`, `UBC-NLP/Simba-W`, `UBC-NLP/Simba-X`, `UBC-NLP/Simba-H`, `UBC-NLP/Simba-M`
+)
+
+##### Load the multilingual African adapter (Only for  `UBC-NLP/Simba-M`)
+asr_pipeline.model.load_adapter("multilingual_african")  # Only for  `UBC-NLP/Simba-M`
+###########################
+
+# Transcribe audio from file
+result = asr_pipeline("https://africa.dlnlp.ai/simba/audio/afr_Lwazi_afr_test_idx3889.wav")
+print(result["text"])
+
+
+# Transcribe audio from audio array
+result = asr_pipeline({
+    "array": audio_array,
+    "sampling_rate": 16_000
+})
+print(result["text"])
+
+```
+
+#### Example Outputs
+
+Using the same audio file with different Simba models:
+
+```python
+# Simba-S
+{'text': 'watter verontwaardiging sou daar, in ons binneste gewees het.'}
+```
+
+```python
+# Simba-W
+{'text': 'watter veronwaardigingsel daar, in ons binneste gewees het.'}
+```
+
+```python
+# Simba-X
+{'text': 'fator fr on ar taamsodr is'}
+```
+
+```python
+# Simba-M
+{'text': 'watter veronwaardiging sodaar in ons binniste gewees het'}
+```
+
+```python
+# Simba-H
+{'text': 'watter vironwaardiging so daar in ons binneste geweeshet'}
+```
+
+Get started with Simba models in minutes using our interactive Colab notebook: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/UBC-NLP/simba/blob/main/simba_models.ipynb)
+
+
+## Citation
+
+If you use the Simba models or SimbaBench  benchmark for your scientific publication, or if you find the resources in this website useful, please cite our paper.
+
+```bibtex
+
+@inproceedings{elmadany-etal-2025-voice,
+    title = "Voice of a Continent: Mapping {A}frica{'}s Speech Technology Frontier",
+    author = "Elmadany, AbdelRahim A.  and
+      Kwon, Sang Yun  and
+      Toyin, Hawau Olamide  and
+      Alcoba Inciarte, Alcides  and
+      Aldarmaki, Hanan  and
+      Abdul-Mageed, Muhammad",
+    editor = "Christodoulopoulos, Christos  and
+      Chakraborty, Tanmoy  and
+      Rose, Carolyn  and
+      Peng, Violet",
+    booktitle = "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing",
+    month = nov,
+    year = "2025",
+    address = "Suzhou, China",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2025.emnlp-main.559/",
+    doi = "10.18653/v1/2025.emnlp-main.559",
+    pages = "11039--11061",
+    ISBN = "979-8-89176-332-6",
+}
+
+```
+

added_tokens.json

@@ -0,0 +1,100 @@
+{
+  "__afr__": 256001,
+  "__amh__": 256002,
+  "__arb__": 256003,
+  "__ary__": 256004,
+  "__arz__": 256005,
+  "__asm__": 256006,
+  "__azj__": 256007,
+  "__bel__": 256008,
+  "__ben__": 256009,
+  "__bos__": 256010,
+  "__bul__": 256011,
+  "__cat__": 256012,
+  "__ceb__": 256013,
+  "__ces__": 256014,
+  "__ckb__": 256015,
+  "__cmn_Hant__": 256017,
+  "__cmn__": 256016,
+  "__cym__": 256018,
+  "__dan__": 256019,
+  "__deu__": 256020,
+  "__ell__": 256021,
+  "__eng__": 256022,
+  "__est__": 256023,
+  "__eus__": 256024,
+  "__fin__": 256025,
+  "__fra__": 256026,
+  "__fuv__": 256027,
+  "__gaz__": 256028,
+  "__gle__": 256029,
+  "__glg__": 256030,
+  "__guj__": 256031,
+  "__heb__": 256032,
+  "__hin__": 256033,
+  "__hrv__": 256034,
+  "__hun__": 256035,
+  "__hye__": 256036,
+  "__ibo__": 256037,
+  "__ind__": 256038,
+  "__isl__": 256039,
+  "__ita__": 256040,
+  "__jav__": 256041,
+  "__jpn__": 256042,
+  "__kan__": 256043,
+  "__kat__": 256044,
+  "__kaz__": 256045,
+  "__khk__": 256046,
+  "__khm__": 256047,
+  "__kir__": 256048,
+  "__kor__": 256049,
+  "__lao__": 256050,
+  "__lit__": 256051,
+  "__lug__": 256052,
+  "__luo__": 256053,
+  "__lvs__": 256054,
+  "__mai__": 256055,
+  "__mal__": 256056,
+  "__mar__": 256057,
+  "__mkd__": 256058,
+  "__mlt__": 256059,
+  "__mni__": 256060,
+  "__mya__": 256061,
+  "__nld__": 256062,
+  "__nno__": 256063,
+  "__nob__": 256064,
+  "__npi__": 256065,
+  "__nya__": 256066,
+  "__ory__": 256067,
+  "__pan__": 256068,
+  "__pbt__": 256069,
+  "__pes__": 256070,
+  "__pol__": 256071,
+  "__por__": 256072,
+  "__ron__": 256073,
+  "__rus__": 256074,
+  "__sat__": 256075,
+  "__slk__": 256076,
+  "__slv__": 256077,
+  "__sna__": 256078,
+  "__snd__": 256079,
+  "__som__": 256080,
+  "__spa__": 256081,
+  "__srp__": 256082,
+  "__swe__": 256083,
+  "__swh__": 256084,
+  "__tam__": 256085,
+  "__tel__": 256086,
+  "__tgk__": 256087,
+  "__tgl__": 256088,
+  "__tha__": 256089,
+  "__tur__": 256090,
+  "__ukr__": 256091,
+  "__urd__": 256092,
+  "__uzn__": 256093,
+  "__vie__": 256094,
+  "__yor__": 256095,
+  "__yue__": 256096,
+  "__zlm__": 256097,
+  "__zul__": 256098
+}

config.json

@@ -0,0 +1,119 @@
+{
+  "_name_or_path": "facebook/seamless-m4t-v2-large",
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "adaptor_dropout": 0.1,
+  "adaptor_kernel_size": 8,
+  "adaptor_stride": 8,
+  "add_adapter": true,
+  "architectures": [
+    "SeamlessM4Tv2ForSpeechToText"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 2,
+  "char_vocab_size": 10943,
+  "conv_depthwise_kernel_size": 31,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 8192,
+  "decoder_layerdrop": 0.05,
+  "decoder_layers": 24,
+  "decoder_start_token_id": 3,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 8192,
+  "encoder_layerdrop": 0.05,
+  "encoder_layers": 24,
+  "eos_token_id": 3,
+  "feature_projection_input_dim": 160,
+  "forced_decoder_ids": null,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "is_encoder_decoder": true,
+  "lang_embed_dim": 256,
+  "layer_norm_eps": 1e-05,
+  "leaky_relu_slope": 0.1,
+  "left_max_position_embeddings": 64,
+  "max_new_tokens": 256,
+  "max_position_embeddings": 4096,
+  "model_type": "seamless_m4t_v2",
+  "num_adapter_layers": 1,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embeddings_type": "relative_key",
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "right_max_position_embeddings": 8,
+  "sampling_rate": 16000,
+  "scale_embedding": true,
+  "speech_encoder_attention_heads": 16,
+  "speech_encoder_chunk_size": 20000,
+  "speech_encoder_dropout": 0.0,
+  "speech_encoder_hidden_act": "swish",
+  "speech_encoder_intermediate_size": 4096,
+  "speech_encoder_layerdrop": 0.1,
+  "speech_encoder_layers": 24,
+  "speech_encoder_left_chunk_num": 128,
+  "spkr_embed_dim": 256,
+  "t2u_bos_token_id": 0,
+  "t2u_decoder_attention_heads": 16,
+  "t2u_decoder_ffn_dim": 8192,
+  "t2u_decoder_layers": 6,
+  "t2u_encoder_attention_heads": 16,
+  "t2u_encoder_ffn_dim": 8192,
+  "t2u_encoder_layers": 6,
+  "t2u_eos_token_id": 2,
+  "t2u_max_position_embeddings": 4096,
+  "t2u_pad_token_id": 1,
+  "t2u_variance_pred_dropout": 0.5,
+  "t2u_variance_predictor_embed_dim": 1024,
+  "t2u_variance_predictor_hidden_dim": 256,
+  "t2u_variance_predictor_kernel_size": 3,
+  "t2u_vocab_size": 10082,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.1",
+  "unit_embed_dim": 1280,
+  "unit_hifi_gan_vocab_size": 10000,
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    11,
+    8,
+    8,
+    4,
+    4
+  ],
+  "upsample_rates": [
+    5,
+    4,
+    4,
+    2,
+    2
+  ],
+  "use_cache": true,
+  "var_pred_dropout": 0.5,
+  "variance_predictor_kernel_size": 3,
+  "vocab_size": 256102,
+  "vocoder_num_langs": 36,
+  "vocoder_num_spkrs": 200,
+  "vocoder_offset": 4
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87ed6293668cbe587d0e04f88291cea44a436b02275aadc637b945d98e16e0da
+size 4999937160

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:942459ecfe7aa127752f7000d2e91188d9278b312b88539e5c6c2ce71b4cf331
+size 1007614496

preprocessor_config.json

@@ -0,0 +1,111 @@
+{
+  "feature_extractor_type": "SeamlessM4TFeatureExtractor",
+  "feature_size": 80,
+  "language_code": [
+    "__afr__",
+    "__amh__",
+    "__arb__",
+    "__ary__",
+    "__arz__",
+    "__asm__",
+    "__azj__",
+    "__bel__",
+    "__ben__",
+    "__bos__",
+    "__bul__",
+    "__cat__",
+    "__ceb__",
+    "__ces__",
+    "__ckb__",
+    "__cmn__",
+    "__cmn_Hant__",
+    "__cym__",
+    "__dan__",
+    "__deu__",
+    "__ell__",
+    "__eng__",
+    "__est__",
+    "__eus__",
+    "__fin__",
+    "__fra__",
+    "__fuv__",
+    "__gaz__",
+    "__gle__",
+    "__glg__",
+    "__guj__",
+    "__heb__",
+    "__hin__",
+    "__hrv__",
+    "__hun__",
+    "__hye__",
+    "__ibo__",
+    "__ind__",
+    "__isl__",
+    "__ita__",
+    "__jav__",
+    "__jpn__",
+    "__kan__",
+    "__kat__",
+    "__kaz__",
+    "__khk__",
+    "__khm__",
+    "__kir__",
+    "__kor__",
+    "__lao__",
+    "__lit__",
+    "__lug__",
+    "__luo__",
+    "__lvs__",
+    "__mai__",
+    "__mal__",
+    "__mar__",
+    "__mkd__",
+    "__mlt__",
+    "__mni__",
+    "__mya__",
+    "__nld__",
+    "__nno__",
+    "__nob__",
+    "__npi__",
+    "__nya__",
+    "__ory__",
+    "__pan__",
+    "__pbt__",
+    "__pes__",
+    "__pol__",
+    "__por__",
+    "__ron__",
+    "__rus__",
+    "__sat__",
+    "__slk__",
+    "__slv__",
+    "__sna__",
+    "__snd__",
+    "__som__",
+    "__spa__",
+    "__srp__",
+    "__swe__",
+    "__swh__",
+    "__tam__",
+    "__tel__",
+    "__tgk__",
+    "__tgl__",
+    "__tha__",
+    "__tur__",
+    "__ukr__",
+    "__urd__",
+    "__uzn__",
+    "__vie__",
+    "__yor__",
+    "__yue__",
+    "__zlm__",
+    "__zul__"
+  ],
+  "num_mel_bins": 80,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "SeamlessM4TProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "stride": 2
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:026a76827537db9f1348e4d5aaa127bb10a2f2ff633243f3a52d16be82d73f9d
+size 5165809

special_tokens_map.json

@@ -0,0 +1,144 @@
+{
+  "additional_special_tokens": [
+    "__afr__",
+    "__amh__",
+    "__arb__",
+    "__ary__",
+    "__arz__",
+    "__asm__",
+    "__azj__",
+    "__bel__",
+    "__ben__",
+    "__bos__",
+    "__bul__",
+    "__cat__",
+    "__ceb__",
+    "__ces__",
+    "__ckb__",
+    "__cmn__",
+    "__cmn_Hant__",
+    "__cym__",
+    "__dan__",
+    "__deu__",
+    "__ell__",
+    "__eng__",
+    "__est__",
+    "__eus__",
+    "__fin__",
+    "__fra__",
+    "__fuv__",
+    "__gaz__",
+    "__gle__",
+    "__glg__",
+    "__guj__",
+    "__heb__",
+    "__hin__",
+    "__hrv__",
+    "__hun__",
+    "__hye__",
+    "__ibo__",
+    "__ind__",
+    "__isl__",
+    "__ita__",
+    "__jav__",
+    "__jpn__",
+    "__kan__",
+    "__kat__",
+    "__kaz__",
+    "__khk__",
+    "__khm__",
+    "__kir__",
+    "__kor__",
+    "__lao__",
+    "__lit__",
+    "__lug__",
+    "__luo__",
+    "__lvs__",
+    "__mai__",
+    "__mal__",
+    "__mar__",
+    "__mkd__",
+    "__mlt__",
+    "__mni__",
+    "__mya__",
+    "__nld__",
+    "__nno__",
+    "__nob__",
+    "__npi__",
+    "__nya__",
+    "__ory__",
+    "__pan__",
+    "__pbt__",
+    "__pes__",
+    "__pol__",
+    "__por__",
+    "__ron__",
+    "__rus__",
+    "__sat__",
+    "__slk__",
+    "__slv__",
+    "__sna__",
+    "__snd__",
+    "__som__",
+    "__spa__",
+    "__srp__",
+    "__swe__",
+    "__swh__",
+    "__tam__",
+    "__tel__",
+    "__tgk__",
+    "__tgl__",
+    "__tha__",
+    "__tur__",
+    "__ukr__",
+    "__urd__",
+    "__uzn__",
+    "__vie__",
+    "__yor__",
+    "__yue__",
+    "__zlm__",
+    "__zul__"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb08273b3ab8f38ebcb33558f96257ef97beeb0441f8a28eb886ae71d9f3e1fa
+size 31219543

tokenizer_config.json

@@ -0,0 +1,935 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256001": {
+      "content": "__afr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256002": {
+      "content": "__amh__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256003": {
+      "content": "__arb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256004": {
+      "content": "__ary__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256005": {
+      "content": "__arz__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256006": {
+      "content": "__asm__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256007": {
+      "content": "__azj__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256008": {
+      "content": "__bel__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256009": {
+      "content": "__ben__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256010": {
+      "content": "__bos__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256011": {
+      "content": "__bul__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256012": {
+      "content": "__cat__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256013": {
+      "content": "__ceb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256014": {
+      "content": "__ces__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256015": {
+      "content": "__ckb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256016": {
+      "content": "__cmn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256017": {
+      "content": "__cmn_Hant__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256018": {
+      "content": "__cym__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256019": {
+      "content": "__dan__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256020": {
+      "content": "__deu__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256021": {
+      "content": "__ell__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256022": {
+      "content": "__eng__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256023": {
+      "content": "__est__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256024": {
+      "content": "__eus__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256025": {
+      "content": "__fin__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256026": {
+      "content": "__fra__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256027": {
+      "content": "__fuv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256028": {
+      "content": "__gaz__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256029": {
+      "content": "__gle__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256030": {
+      "content": "__glg__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256031": {
+      "content": "__guj__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256032": {
+      "content": "__heb__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256033": {
+      "content": "__hin__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256034": {
+      "content": "__hrv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256035": {
+      "content": "__hun__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256036": {
+      "content": "__hye__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256037": {
+      "content": "__ibo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256038": {
+      "content": "__ind__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256039": {
+      "content": "__isl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256040": {
+      "content": "__ita__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256041": {
+      "content": "__jav__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256042": {
+      "content": "__jpn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256043": {
+      "content": "__kan__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256044": {
+      "content": "__kat__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256045": {
+      "content": "__kaz__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256046": {
+      "content": "__khk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256047": {
+      "content": "__khm__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256048": {
+      "content": "__kir__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256049": {
+      "content": "__kor__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256050": {
+      "content": "__lao__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256051": {
+      "content": "__lit__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256052": {
+      "content": "__lug__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256053": {
+      "content": "__luo__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256054": {
+      "content": "__lvs__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256055": {
+      "content": "__mai__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256056": {
+      "content": "__mal__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256057": {
+      "content": "__mar__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256058": {
+      "content": "__mkd__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256059": {
+      "content": "__mlt__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256060": {
+      "content": "__mni__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256061": {
+      "content": "__mya__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256062": {
+      "content": "__nld__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256063": {
+      "content": "__nno__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256064": {
+      "content": "__nob__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256065": {
+      "content": "__npi__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256066": {
+      "content": "__nya__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256067": {
+      "content": "__ory__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256068": {
+      "content": "__pan__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256069": {
+      "content": "__pbt__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256070": {
+      "content": "__pes__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256071": {
+      "content": "__pol__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256072": {
+      "content": "__por__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256073": {
+      "content": "__ron__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256074": {
+      "content": "__rus__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256075": {
+      "content": "__sat__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256076": {
+      "content": "__slk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256077": {
+      "content": "__slv__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256078": {
+      "content": "__sna__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256079": {
+      "content": "__snd__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256080": {
+      "content": "__som__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256081": {
+      "content": "__spa__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256082": {
+      "content": "__srp__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256083": {
+      "content": "__swe__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256084": {
+      "content": "__swh__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256085": {
+      "content": "__tam__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256086": {
+      "content": "__tel__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256087": {
+      "content": "__tgk__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256088": {
+      "content": "__tgl__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256089": {
+      "content": "__tha__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256090": {
+      "content": "__tur__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256091": {
+      "content": "__ukr__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256092": {
+      "content": "__urd__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256093": {
+      "content": "__uzn__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256094": {
+      "content": "__vie__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256095": {
+      "content": "__yor__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256096": {
+      "content": "__yue__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256097": {
+      "content": "__zlm__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256098": {
+      "content": "__zul__",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "__afr__",
+    "__amh__",
+    "__arb__",
+    "__ary__",
+    "__arz__",
+    "__asm__",
+    "__azj__",
+    "__bel__",
+    "__ben__",
+    "__bos__",
+    "__bul__",
+    "__cat__",
+    "__ceb__",
+    "__ces__",
+    "__ckb__",
+    "__cmn__",
+    "__cmn_Hant__",
+    "__cym__",
+    "__dan__",
+    "__deu__",
+    "__ell__",
+    "__eng__",
+    "__est__",
+    "__eus__",
+    "__fin__",
+    "__fra__",
+    "__fuv__",
+    "__gaz__",
+    "__gle__",
+    "__glg__",
+    "__guj__",
+    "__heb__",
+    "__hin__",
+    "__hrv__",
+    "__hun__",
+    "__hye__",
+    "__ibo__",
+    "__ind__",
+    "__isl__",
+    "__ita__",
+    "__jav__",
+    "__jpn__",
+    "__kan__",
+    "__kat__",
+    "__kaz__",
+    "__khk__",
+    "__khm__",
+    "__kir__",
+    "__kor__",
+    "__lao__",
+    "__lit__",
+    "__lug__",
+    "__luo__",
+    "__lvs__",
+    "__mai__",
+    "__mal__",
+    "__mar__",
+    "__mkd__",
+    "__mlt__",
+    "__mni__",
+    "__mya__",
+    "__nld__",
+    "__nno__",
+    "__nob__",
+    "__npi__",
+    "__nya__",
+    "__ory__",
+    "__pan__",
+    "__pbt__",
+    "__pes__",
+    "__pol__",
+    "__por__",
+    "__ron__",
+    "__rus__",
+    "__sat__",
+    "__slk__",
+    "__slv__",
+    "__sna__",
+    "__snd__",
+    "__som__",
+    "__spa__",
+    "__srp__",
+    "__swe__",
+    "__swh__",
+    "__tam__",
+    "__tel__",
+    "__tgk__",
+    "__tgl__",
+    "__tha__",
+    "__tur__",
+    "__ukr__",
+    "__urd__",
+    "__uzn__",
+    "__vie__",
+    "__yor__",
+    "__yue__",
+    "__zlm__",
+    "__zul__"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "SeamlessM4TProcessor",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "src_lang": "__eng__",
+  "tgt_lang": "__fra__",
+  "tokenizer_class": "SeamlessM4TTokenizer",
+  "unk_token": "<unk>"
+}