Upload configuration_neurovoice.py with huggingface_hub

configuration_neurovoice.py

@@ -0,0 +1,80 @@
+import os
+import torch
+from transformers import PretrainedConfig, AutoConfig
+
+
+class NeuroVoiceConfig(PretrainedConfig):
+    """
+    Configuration class for NeuroVoice-0.5B Neural Audio Language Model.
+    Couples a Qwen2.5-0.5B backbone with Kyutai Mimi 24kHz audio codec.
+    """
+    model_type = "neurovoice"
+
+    def __init__(
+        self,
+        text_vocab_size: int = 151936,
+        audio_vocab_size: int = 2048,
+        num_codebooks: int = 8,
+        d_model: int = 896,
+        num_heads: int = 14,
+        num_kv_heads: int = 2,
+        num_layers: int = 24,
+        num_depth_layers: int = 4,
+        d_ff: int = 4864,
+        max_seq_len: int = 2048,
+        rope_theta: float = 1000000.0,
+        dropout_rate: float = 0.0,
+        activation: str = "swiglu",
+        dtype: str = "bfloat16",
+        use_qk_norm: bool = False,
+        pad_token_id: int = 0,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        unk_token_id: int = 3,
+        instruct_token_id: int = 4,
+        text_token_id: int = 5,
+        ref_audio_token_id: int = 6,
+        audio_start_token_id: int = 7,
+        audio_end_token_id: int = 8,
+        **kwargs,
+    ):
+        self.text_vocab_size = text_vocab_size
+        self.audio_vocab_size = audio_vocab_size
+        self.num_codebooks = num_codebooks
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads
+        self.num_layers = num_layers
+        self.num_depth_layers = num_depth_layers
+        self.d_ff = d_ff
+        self.max_seq_len = max_seq_len
+        self.rope_theta = rope_theta
+        self.dropout_rate = dropout_rate
+        self.activation = activation
+        self.dtype = dtype
+        self.use_qk_norm = use_qk_norm
+
+        self.instruct_token_id = instruct_token_id
+        self.text_token_id = text_token_id
+        self.ref_audio_token_id = ref_audio_token_id
+        self.audio_start_token_id = audio_start_token_id
+        self.audio_end_token_id = audio_end_token_id
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            unk_token_id=unk_token_id,
+            **kwargs,
+        )
+
+    @property
+    def torch_dtype(self) -> torch.dtype:
+        if self.dtype == "bfloat16":
+            return torch.bfloat16
+        elif self.dtype == "float16":
+            return torch.float16
+        return torch.float32
+
+
+AutoConfig.register("neurovoice", NeuroVoiceConfig)