Initial commit

README.md

@@ -0,0 +1,138 @@
+---
+library_name: transformers
+license: cc-by-nc-4.0
+pipeline_tag: feature-extraction
+tags:
+- audio
+- codec
+---
+
+# X-Codec2 (Transformers-native)
+
+The X-Codec2 model was proposed in [Llasa: Scaling Train-Time and Inference-Time Compute for Llama-based Speech Synthesis](https://huggingface.co/papers/2502.04128).
+
+X-Codec2 is a neural audio codec designed to improve speech synthesis and general audio generation for large language model (LLM) pipelines. It extends the original X-Codec by refining how semantic and acoustic information is integrated and tokenized, enabling efficient and high-fidelity audio representation.
+
+About its architecture:
+- **Unified Semantic-Acoustic Tokenization**: X-Codec2 fuses outputs from a semantic encoder (e.g., Wav2Vec2-BERT) and an acoustic encoder into a single embedding, capturing both high-level meaning (e.g., text content, emotion) and low-level audio details (e.g., timbre).
+- **Single-Stage Feature Scalar Quantization (FSQ)**: Unlike the multi-layer residual VQ in most approaches (e.g., DAC, EnCodec, X-Codec, Mimi), X-Codec2 uses a single-layer of Feature Scalar Quantization (FSQ) for stability and compatibility with causal, autoregressive LLMs.
+- **Transformer-Friendly Design**: The 1D token structure of X-Codec2 naturally aligns with the autoregressive modeling in LLMs like LLaMA, improving training efficiency and downstream compatibility.
+
+This model was contributed by [Eric Bezzam](https://huggingface.co/bezzam) and [Steven Zheng](https://huggingface.co/Steveeeeeeen).
+The original modeling code can be found [here](https://huggingface.co/HKUSTAudio/xcodec2/blob/main/modeling_xcodec2.py), while their training code is [here](https://github.com/zhenye234/X-Codec-2.0).
+
+
+## Setup
+
+X-Codec2 is supported natively in 🤗 Transformers. Until it is part of an official Transformers release, install from source:
+
+```bash
+pip install git+https://github.com/huggingface/transformers
+```
+
+## Usage example
+
+Here is a quick example of how to encode and decode an audio using this model:
+
+```python
+from datasets import Audio, load_dataset
+from transformers import AutoFeatureExtractor, AutoModel
+
+model_id = "HKUSTAudio/xcodec2-hf"
+model = AutoModel.from_pretrained(model_id, device_map="auto")
+feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
+
+dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate))
+audio = dataset[0]["audio"]["array"]
+inputs = feature_extractor(audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt").to(
+    model.device, model.dtype
+)
+print("Input waveform shape:", inputs["input_values"].shape)
+# Input waveform shape: torch.Size([1, 1, 93760])
+
+# encoder and decoder
+audio_codes = model.encode(**inputs).audio_codes
+print("Audio codes shape:", audio_codes.shape)
+# Audio codes shape: torch.Size([1, 1, 293])
+audio_values = model.decode(audio_codes).audio_values
+print("Audio values shape:", audio_values.shape)
+# Audio values shape: torch.Size([1, 1, 93760])
+
+# Equivalently, you can do encoding and decoding in one step
+model_output = model(**inputs)
+audio_codes = model_output.audio_codes
+audio_values = model_output.audio_values
+```
+
+### Batch processing
+
+Unlike the original [release](https://huggingface.co/HKUSTAudio/xcodec2), this implementation also supports batched inputs.
+
+```python
+from datasets import Audio, load_dataset
+from transformers import AutoFeatureExtractor, AutoModel
+
+batch_size = 2
+model_id = "HKUSTAudio/xcodec2-hf"
+model = AutoModel.from_pretrained(model_id, device_map="auto")
+feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
+
+dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate))
+audios = [dataset[i]["audio"]["array"] for i in range(batch_size)]
+inputs = feature_extractor(audio=audios, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt").to(
+    model.device, model.dtype
+)
+print("Input waveform shape:", inputs["input_values"].shape)
+# Input waveform shape: torch.Size([2, 1, 93760])
+
+# encoder and decoder
+encoder_output = model.encode(**inputs)
+audio_codes = encoder_output.audio_codes
+print("Audio codes shape:", audio_codes.shape)
+# Audio codes shape: torch.Size([2, 1, 293])
+audio_values = model.decode(audio_codes).audio_values
+print("Audio values shape:", audio_values.shape)
+# Audio values shape: torch.Size([2, 1, 93760])
+
+# Equivalently, you can do encoding and decoding in one step
+model_output = model(**inputs)
+audio_codes = model_output.audio_codes
+audio_values = model_output.audio_values
+```
+
+### Speed-up with `torch.compile`
+
+You can speed up inference with [`torch.compile`](https://pytorch.org/docs/stable/generated/torch.compile.html). The first few calls will be slower due to compilation overhead, but subsequent calls will be faster.
+
+On an A100, we observed a speed-up of ~1.35 for a batch size of 4 ([script](https://gist.github.com/ebezzam/3b79481b5d48d8e35c4ecc582aee0cb3#file-benchmark_torch_compile-py)).
+
+```python
+import torch
+from datasets import Audio, load_dataset
+from transformers import AutoFeatureExtractor, AutoModel
+
+batch_size = 4
+model_id = "HKUSTAudio/xcodec2-hf"
+model = AutoModel.from_pretrained(model_id, device_map="auto")
+feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
+
+dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate))
+audios = [dataset[i]["audio"]["array"] for i in range(batch_size)]
+inputs = feature_extractor(
+    audio=audios, sampling_rate=feature_extractor.sampling_rate, padding=True, return_tensors="pt"
+).to(model.device, model.dtype)
+
+compiled_model = torch.compile(model, fullgraph=True)
+
+# Warmup (includes compilation on first call)
+for _ in range(10):
+    with torch.inference_mode():
+        _ = compiled_model(**inputs)
+
+with torch.inference_mode():
+    output = compiled_model(**inputs)
+print("Audio values shape:", output.audio_values.shape)
+```

config.json

@@ -0,0 +1,128 @@
+{
+  "activation_dropout": 0.1,
+  "architectures": [
+    "Xcodec2Model"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "downsampling_ratios": [
+    2,
+    2,
+    4,
+    4,
+    5
+  ],
+  "dtype": "float32",
+  "encoder_hidden_size": 48,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "max_position_embeddings": 4096,
+  "model_type": "xcodec2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 16,
+  "pad_token_id": null,
+  "quantization_dim": 2048,
+  "quantization_levels": [
+    4,
+    4,
+    4,
+    4,
+    4,
+    4,
+    4,
+    4
+  ],
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sampling_rate": 16000,
+  "semantic_model_config": {
+    "_name_or_path": "facebook/w2v-bert-2.0",
+    "activation_dropout": 0.0,
+    "adapter_act": "relu",
+    "adapter_kernel_size": 3,
+    "adapter_stride": 2,
+    "add_adapter": false,
+    "apply_spec_augment": false,
+    "architectures": [
+      "Wav2Vec2BertModel"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "classifier_proj_size": 768,
+    "codevector_dim": 768,
+    "conformer_conv_dropout": 0.1,
+    "contrastive_logits_temperature": 0.1,
+    "conv_depthwise_kernel_size": 31,
+    "ctc_loss_reduction": "sum",
+    "ctc_zero_infinity": false,
+    "diversity_loss_weight": 0.1,
+    "dtype": "float32",
+    "eos_token_id": 2,
+    "feat_proj_dropout": 0.0,
+    "feat_quantizer_dropout": 0.0,
+    "feature_projection_input_dim": 160,
+    "final_dropout": 0.1,
+    "hidden_act": "swish",
+    "hidden_dropout": 0.0,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-05,
+    "layerdrop": 0.1,
+    "left_max_position_embeddings": 64,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 5000,
+    "model_type": "wav2vec2-bert",
+    "num_adapter_layers": 1,
+    "num_attention_heads": 16,
+    "num_codevector_groups": 2,
+    "num_codevectors_per_group": 320,
+    "num_hidden_layers": 16,
+    "num_negatives": 100,
+    "output_hidden_size": 1024,
+    "pad_token_id": 0,
+    "position_embeddings_type": "relative_key",
+    "proj_codevector_dim": 768,
+    "right_max_position_embeddings": 8,
+    "rotary_embedding_base": 10000,
+    "tdnn_dilation": [
+      1,
+      2,
+      3,
+      1,
+      1
+    ],
+    "tdnn_dim": [
+      512,
+      512,
+      512,
+      512,
+      1500
+    ],
+    "tdnn_kernel": [
+      5,
+      3,
+      3,
+      1,
+      1
+    ],
+    "use_intermediate_ffn_before_adapter": false,
+    "use_weighted_layer_sum": false,
+    "vocab_size": null,
+    "xvector_output_dim": 512
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.13.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:611a63e4dff70c19bd4718d701bb7bc522acf6293a109ab62f5db2f7ff395114
+size 2517231448

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "feature_extractor_type": "Xcodec2FeatureExtractor",
+  "feature_size": 80,
+  "frame_length": 400,
+  "frame_shift": 160,
+  "hop_length": 320,
+  "num_mel_bins": 80,
+  "padding_side": "right",
+  "padding_value": 1,
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "stride": 2
+}