Upload Mongolian Moonshine ASR (WER 11.88 pct)

config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "MoonshineForConditionalGeneration"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "decoder_hidden_act": "silu",
+  "decoder_num_attention_heads": 8,
+  "decoder_num_hidden_layers": 8,
+  "decoder_num_key_value_heads": 8,
+  "decoder_start_token_id": 1,
+  "dtype": "float32",
+  "encoder_hidden_act": "gelu",
+  "encoder_num_attention_heads": 8,
+  "encoder_num_hidden_layers": 8,
+  "encoder_num_key_value_heads": 8,
+  "eos_token_id": 2,
+  "hidden_size": 416,
+  "initializer_range": 0.02,
+  "intermediate_size": 1664,
+  "is_encoder_decoder": true,
+  "max_position_embeddings": 194,
+  "model_type": "moonshine",
+  "pad_head_dim_to_multiple_of": 8,
+  "pad_token_id": 2,
+  "partial_rotary_factor": 0.62,
+  "rope_parameters": {
+    "partial_rotary_factor": 0.62,
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.12.1",
+  "use_cache": false,
+  "vocab_size": 2003
+}

generation_config.json

@@ -0,0 +1,17 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "decoder_start_token_id": 1,
+  "do_sample": false,
+  "early_stopping": true,
+  "eos_token_id": [
+    2
+  ],
+  "length_penalty": 1.2,
+  "max_length": 194,
+  "no_repeat_ngram_size": 2,
+  "num_beams": 5,
+  "pad_token_id": 2,
+  "repetition_penalty": 1.2,
+  "transformers_version": "5.12.1"
+}

mn_bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23510d1ccd07f7219758f14f32eca9b542e9af6c7c03f6e16c25d898dac4107
+size 37637

mn_tokenizer.py

@@ -0,0 +1,82 @@
+"""
+MnBPETokenizer - HuggingFace PreTrainedTokenizer for the Mongolian BPE model.
+
+Special token layout (matches training in mn_tokenizer_patch.py):
+    id 0 -> <pad>
+    id 1 -> <s>   BOS
+    id 2 -> </s>  EOS / PAD
+    id 3+ -> BPE pieces (offset = 3)
+"""
+import os, shutil
+from typing import Dict, List, Optional, Tuple
+import sentencepiece as spm
+from transformers import PreTrainedTokenizer
+
+VOCAB_FILES_NAMES = {"vocab_file": "mn_bpe.model"}
+_SPECIAL = {0: "<pad>", 1: "<s>", 2: "</s>"}
+_OFFSET = 3
+
+
+class MnBPETokenizer(PreTrainedTokenizer):
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(self, vocab_file, bos_token="<s>", eos_token="</s>",
+                 unk_token="<unk>", pad_token="</s>",
+                 sp_model_kwargs=None, **kwargs):
+        self.sp_model_kwargs = sp_model_kwargs or {}
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self.vocab_file = vocab_file
+        super().__init__(bos_token=bos_token, eos_token=eos_token,
+                         unk_token=unk_token, pad_token=pad_token,
+                         sp_model_kwargs=sp_model_kwargs, **kwargs)
+
+    @property
+    def vocab_size(self):
+        return self.sp_model.get_piece_size() + _OFFSET
+
+    def get_vocab(self):
+        v = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        v.update(self.added_tokens_encoder)
+        return v
+
+    def _tokenize(self, text):
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        rev = {v: k for k, v in _SPECIAL.items()}
+        if token in rev:
+            return rev[token]
+        return self.sp_model.piece_to_id(token) + _OFFSET
+
+    def _convert_id_to_token(self, index):
+        if index in _SPECIAL:
+            return _SPECIAL[index]
+        return self.sp_model.id_to_piece(index - _OFFSET)
+
+    def convert_tokens_to_string(self, tokens):
+        return self.sp_model.decode(tokens)
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        if not os.path.isdir(save_directory):
+            return ()
+        fname = VOCAB_FILES_NAMES["vocab_file"]
+        if filename_prefix:
+            fname = f"{filename_prefix}-{fname}"
+        out = os.path.join(save_directory, fname)
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out):
+            shutil.copyfile(self.vocab_file, out)
+        return (out,)
+
+    def decode_ids(self, ids, skip_special=True):
+        """Decode token ids to text, matching training decode logic."""
+        pieces = []
+        for i in ids:
+            i = int(i)
+            if i == self.eos_token_id:
+                break
+            if skip_special and i < _OFFSET:
+                continue
+            pieces.append(i - _OFFSET)
+        return self.sp_model.decode(pieces) if pieces else ""

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e7aa75168859df3714a3d0de883deca658e1545062d1268c95e25ae43656f1b
+size 194886848

preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": false,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

tokenizer_config.json

@@ -0,0 +1,36 @@
+{
+  "added_tokens_decoder": {
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "backend": "custom",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "MnBPETokenizer",
+  "unk_token": "<unk>"
+}