Feature Extraction
Transformers
Safetensors
liquid_codec
liquid
lfm3
audio
audio-codec
speech
streaming
edge
multilingual
custom_code
Instructions to use eustlb/Liquid-Codec with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use eustlb/Liquid-Codec with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("feature-extraction", model="eustlb/Liquid-Codec", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("eustlb/Liquid-Codec", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Mirror LiquidAI/Liquid-Codec @ b5abfac4
Browse files- README.md +235 -4
- config.json +32 -89
- configuration_lfm3.py +103 -0
- configuration_liquid_codec.py +118 -0
- modeling_lfm3.py +658 -0
- modeling_liquid_codec.py +340 -0
- preprocessor_config.json +0 -12
README.md
CHANGED
|
@@ -1,13 +1,244 @@
|
|
| 1 |
---
|
|
|
|
| 2 |
license: other
|
| 3 |
license_name: lfm1.0
|
| 4 |
license_link: LICENSE
|
| 5 |
-
|
| 6 |
tags:
|
|
|
|
|
|
|
| 7 |
- audio
|
| 8 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
-
# LFM3-Audio-2B codec
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
library_name: transformers
|
| 3 |
license: other
|
| 4 |
license_name: lfm1.0
|
| 5 |
license_link: LICENSE
|
| 6 |
+
pipeline_tag: feature-extraction
|
| 7 |
tags:
|
| 8 |
+
- liquid
|
| 9 |
+
- lfm3
|
| 10 |
- audio
|
| 11 |
+
- audio-codec
|
| 12 |
+
- speech
|
| 13 |
+
- streaming
|
| 14 |
+
- edge
|
| 15 |
+
- multilingual
|
| 16 |
+
- custom_code
|
| 17 |
+
language:
|
| 18 |
+
- en
|
| 19 |
+
- zh
|
| 20 |
+
- de
|
| 21 |
+
- fr
|
| 22 |
+
- ja
|
| 23 |
+
- ko
|
| 24 |
---
|
| 25 |
|
|
|
|
| 26 |
|
| 27 |
+
<div align="center">
|
| 28 |
+
<img
|
| 29 |
+
src="https://huggingface.co/LiquidAI/LFM3-1.7B/resolve/main/assets/liquid-ai-banner.webp"
|
| 30 |
+
alt="Liquid AI"
|
| 31 |
+
style="width: 100%; max-width: 100%; height: auto; display: block; margin: 0;"
|
| 32 |
+
/>
|
| 33 |
+
<div align="center" style="display: flex; justify-content: center; align-items: center; gap: 1.25em; margin: 8px 0 1.7rem; padding: 0; line-height: 1.3;">
|
| 34 |
+
<a href="https://console.liquid.ai/dashboard" style="display: inline-flex; align-items: center; gap: 6px;"><img src="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 width=%2216%22 height=%2216%22 viewBox=%220 0 24 24%22 fill=%22none%22 stroke-width=%222.2%22 stroke-linecap=%22round%22 stroke-linejoin=%22round%22%3E%3Cstyle%3Epath%7Bstroke:%23000%7D%40media (prefers-color-scheme:dark)%7Bpath%7Bstroke:%23d1d5dc%7D%7D%3C/style%3E%3Cpath d=%22m8 6-6 6 6 6m8-12 6 6-6 6m-2-14-4 16%22/%3E%3C/svg%3E" width="16" height="16" alt="" aria-hidden="true" style="display: block; flex: 0 0 16px; width: 16px; height: 16px; margin: 0;" /><strong>API</strong></a>
|
| 35 |
+
<a href="https://playground.liquid.ai/talk" style="display: inline-flex; align-items: center; gap: 6px;"><img src="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 width=%2216%22 height=%2216%22 viewBox=%220 0 24 24%22 fill=%22none%22 stroke-width=%222.2%22 stroke-linecap=%22round%22 stroke-linejoin=%22round%22%3E%3Cstyle%3Epath%7Bstroke:%23000%7D%40media (prefers-color-scheme:dark)%7Bpath%7Bstroke:%23d1d5dc%7D%7D%3C/style%3E%3Cpath transform=%22translate(3 0)%22 d=%22M8 5.5v13L19 12 8 5.5Z%22/%3E%3C/svg%3E" width="16" height="16" alt="" aria-hidden="true" style="display: block; flex: 0 0 16px; width: 16px; height: 16px; margin: 0;" /><strong>Try</strong></a>
|
| 36 |
+
<a href="https://docs.liquid.ai/lfm/getting-started/welcome" style="display: inline-flex; align-items: center; gap: 6px;"><img src="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 width=%2216%22 height=%2216%22 viewBox=%220 0 24 24%22 fill=%22none%22 stroke-width=%222.2%22 stroke-linecap=%22round%22 stroke-linejoin=%22round%22%3E%3Cstyle%3Epath%7Bstroke:%23000%7D%40media (prefers-color-scheme:dark)%7Bpath%7Bstroke:%23d1d5dc%7D%7D%3C/style%3E%3Cpath d=%22M12 6c-3-2-6-2.5-10-2v15c4-.5 7 0 10 2m0-15c3-2 6-2.5 10-2v15c-4-.5-7 0-10 2V6Z%22/%3E%3C/svg%3E" width="16" height="16" alt="" aria-hidden="true" style="display: block; flex: 0 0 16px; width: 16px; height: 16px; margin: 0;" /><strong>Docs</strong></a>
|
| 37 |
+
<a href="https://www.liquid.ai/blog/liquidcodec" style="display: inline-flex; align-items: center; gap: 6px;"><img src="data:image/svg+xml,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 width=%2216%22 height=%2216%22 viewBox=%220 0 24 24%22 fill=%22none%22 stroke-width=%222.2%22 stroke-linecap=%22round%22 stroke-linejoin=%22round%22%3E%3Cstyle%3Epath%7Bstroke:%23000%7D%40media (prefers-color-scheme:dark)%7Bpath%7Bstroke:%23d1d5dc%7D%7D%3C/style%3E%3Cpath transform=%22translate(1 0)%22 d=%22m16 3 5 5L8 21H3v-5L16 3Zm-3 3 5 5%22/%3E%3C/svg%3E" width="16" height="16" alt="" aria-hidden="true" style="display: block; flex: 0 0 16px; width: 16px; height: 16px; margin: 0;" /><strong>Blog</strong></a>
|
| 38 |
+
</div>
|
| 39 |
+
</div>
|
| 40 |
+
|
| 41 |
+
# LiquidCodec
|
| 42 |
+
|
| 43 |
+
LiquidCodec is a **compact, streaming audio codec** that compresses speech into discrete tokens and converts them back into audio. It is the audio tokenizer behind [LFM3-Audio-2B](https://huggingface.co/LiquidAI/LFM3-Audio-2B) and [LFM3-Omni-2B](https://huggingface.co/LiquidAI/LFM3-Omni-2B).
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
- **Easy to predict:** LiquidCodec’s grouped finite scalar quantization (GFSQ) produces codebooks with no sequential dependencies, so speech models can predict all codebooks of a frame in parallel.
|
| 47 |
+
- **Fast:** at FP32 on a single CPU thread of an Apple M4 Max, LiquidCodec encodes at about **361× real time** and decodes at about **87× real time**.
|
| 48 |
+
- **High fidelity:** best ViSQOL and lowest mel distance of all compared codecs tested on eight reconstruction test sets.
|
| 49 |
+
- **Better speech generation:** in matched TTS experiments, models trained on LiquidCodec tokens reach the highest naturalness (UTMOSv2) on all eight test sets.
|
| 50 |
+
- **Edge-ready:** decoder-only quantization-aware training lets the decoder run in Q4_0 with a 3.5× smaller footprint and minimal quality loss.
|
| 51 |
+
|
| 52 |
+
Find more information in [our blog post](https://www.liquid.ai/blog/liquidcodec).
|
| 53 |
+
|
| 54 |
+
## Model details
|
| 55 |
+
<hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
|
| 56 |
+
|
| 57 |
+
<table style="width: 100%; display: table; border: 0; border-radius: 0; border-collapse: collapse;">
|
| 58 |
+
<thead style="border: 0;">
|
| 59 |
+
<tr style="border: 0;">
|
| 60 |
+
<th scope="col" style="text-align: left; padding: 0.55em 0.75em; border: 0; border-bottom: 1px solid rgba(127, 127, 127, 0.55);">Property</th>
|
| 61 |
+
<th scope="col" style="text-align: left; padding: 0.55em 0.75em; border: 0; border-bottom: 1px solid rgba(127, 127, 127, 0.55);">Specification</th>
|
| 62 |
+
</tr>
|
| 63 |
+
</thead>
|
| 64 |
+
<tbody style="border: 0;">
|
| 65 |
+
<tr style="border: 0;">
|
| 66 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Parameters</strong></th>
|
| 67 |
+
<td style="border: 0; padding: 0.55em 0.75em;">~74M</td>
|
| 68 |
+
</tr>
|
| 69 |
+
<tr style="border: 0;">
|
| 70 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Architecture</strong></th>
|
| 71 |
+
<td style="border: 0; padding: 0.55em 0.75em;">Symmetric, fully causal LFM3 encoder and decoder with grouped finite scalar quantization (GFSQ)</td>
|
| 72 |
+
</tr>
|
| 73 |
+
<tr style="border: 0;">
|
| 74 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Input / output</strong></th>
|
| 75 |
+
<td style="border: 0; padding: 0.55em 0.75em;">24 kHz mono speech</td>
|
| 76 |
+
</tr>
|
| 77 |
+
<tr style="border: 0;">
|
| 78 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Frame rate</strong></th>
|
| 79 |
+
<td style="border: 0; padding: 0.55em 0.75em;">12.5 Hz</td>
|
| 80 |
+
</tr>
|
| 81 |
+
<tr style="border: 0;">
|
| 82 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Tokens per frame</strong></th>
|
| 83 |
+
<td style="border: 0; padding: 0.55em 0.75em;">12 (one per codebook)</td>
|
| 84 |
+
</tr>
|
| 85 |
+
<tr style="border: 0;">
|
| 86 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Bitrate</strong></th>
|
| 87 |
+
<td style="border: 0; padding: 0.55em 0.75em;">≈ 1.65 kbps</td>
|
| 88 |
+
</tr>
|
| 89 |
+
<tr style="border: 0;">
|
| 90 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Streaming</strong></th>
|
| 91 |
+
<td style="border: 0; padding: 0.55em 0.75em;">Streaming encoding and decoding</td>
|
| 92 |
+
</tr>
|
| 93 |
+
<tr style="border: 0;">
|
| 94 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>Training data</strong></th>
|
| 95 |
+
<td style="border: 0; padding: 0.55em 0.75em;">264k hours across 107 languages</td>
|
| 96 |
+
</tr>
|
| 97 |
+
<tr style="border: 0;">
|
| 98 |
+
<th scope="row" style="text-align: left; background-color: transparent; font-weight: 400; border: 0; padding: 0.55em 0.75em;"><strong>License</strong></th>
|
| 99 |
+
<td style="border: 0; padding: 0.55em 0.75em;"><a href="https://huggingface.co/LiquidAI/Liquid-Codec/blob/main/LICENSE">LFM Open License v1.0</a></td>
|
| 100 |
+
</tr>
|
| 101 |
+
</tbody>
|
| 102 |
+
</table>
|
| 103 |
+
|
| 104 |
+
LiquidCodec is trained and evaluated on speech. Its quality on music, singing or environmental sound has not been evaluated.
|
| 105 |
+
|
| 106 |
+
## Inference
|
| 107 |
+
<hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
|
| 108 |
+
|
| 109 |
+
This repository contains the BF16 codec weights for use with Transformers. Quantized versions of the decoder (Q4_0 / Q4_K_M / Q8_0) are not published separately; they ship inside [LFM3-Audio-2B-GGUF](https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF) and [LFM3-Omni-2B-GGUF](https://huggingface.co/LiquidAI/LFM3-Omni-2B-GGUF) for on-device inference with [llama.cpp](https://docs.liquid.ai/deployment/on-device/llama-cpp).
|
| 110 |
+
|
| 111 |
+
### Remote-code loading
|
| 112 |
+
|
| 113 |
+
The repository includes the codec configuration and model implementation, so it can be loaded with `AutoModel`:
|
| 114 |
+
|
| 115 |
+
```python
|
| 116 |
+
import torch
|
| 117 |
+
from transformers import AutoModel
|
| 118 |
+
|
| 119 |
+
codec = AutoModel.from_pretrained(
|
| 120 |
+
"LiquidAI/Liquid-Codec",
|
| 121 |
+
trust_remote_code=True,
|
| 122 |
+
dtype=torch.bfloat16,
|
| 123 |
+
device_map="cuda",
|
| 124 |
+
)
|
| 125 |
+
waveform = torch.randn(1, 24_000 * 3, device="cuda") # 24 kHz
|
| 126 |
+
codes = codec.encode(waveform).audio_codes
|
| 127 |
+
reconstructed = codec.decode(codes).audio_values
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
### Native loading and streaming
|
| 131 |
+
|
| 132 |
+
```python
|
| 133 |
+
import torch
|
| 134 |
+
from transformers import LiquidCodecModel, LiquidCodecStreamingCache
|
| 135 |
+
|
| 136 |
+
codec = LiquidCodecModel.from_pretrained("LiquidAI/Liquid-Codec", dtype=torch.bfloat16, device_map="cuda")
|
| 137 |
+
|
| 138 |
+
waveform = torch.randn(1, 24_000 * 3, device="cuda") # 24 kHz
|
| 139 |
+
codes = codec.encode(waveform).audio_codes # (1, 12, 38)
|
| 140 |
+
reconstructed = codec.decode(codes).audio_values # (1, 38 * 1920), float32
|
| 141 |
+
|
| 142 |
+
# streaming synthesis: every chunk returns the samples that no later frame changes any more
|
| 143 |
+
cache = LiquidCodecStreamingCache(codec.config)
|
| 144 |
+
chunks = [codec.decode(codes[..., i : i + 1], cache, finalize=False).audio_values for i in range(codes.shape[-1])]
|
| 145 |
+
chunks.append(codec.decode(codes[..., :0], cache, finalize=True).audio_values)
|
| 146 |
+
streamed = torch.cat(chunks, dim=-1)
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
## Performance
|
| 150 |
+
<hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
|
| 151 |
+
|
| 152 |
+
### Speed
|
| 153 |
+
|
| 154 |
+
FP32, single-threaded CPU inference on a MacBook Pro with an Apple M4 Max. RTF = processing time ÷ audio duration (lower is faster).
|
| 155 |
+
|
| 156 |
+
| Tokenizer | Codebooks | Enc. RTF ↓ | Dec. RTF ↓ |
|
| 157 |
+
|---|---|---|---|
|
| 158 |
+
| **LiquidCodec** | 12 | **0.0028 ± 0.0001** | **0.0115 ± 0.0001** |
|
| 159 |
+
| X-Codec 2 | 1 | 0.7359 ± 0.0103 | 0.0379 ± 0.0009 |
|
| 160 |
+
| NanoCodec | 13 | 0.2145 ± 0.0057 | 0.2543 ± 0.0061 |
|
| 161 |
+
| Mimi | 12 (truncated) | 0.0361 ± 0.0004 | 0.0329 ± 0.0005 |
|
| 162 |
+
|
| 163 |
+
### Reconstruction and zero-shot TTS
|
| 164 |
+
|
| 165 |
+
LiquidCodec achieves the best ViSQOL and lowest mel distance on all eight reconstruction test sets, and remains the best on both with a Q4_0 decoder. TTS models trained on LiquidCodec tokens reach the highest UTMOSv2 on all eight test sets and the best WER/CER on four of eight.
|
| 166 |
+
|
| 167 |
+
| Test set | Tokenizer | SQMOS ↑ | PESQ (WB) ↑ | Mel dist. ↓ | SECS (ECAPA2) ↑ | ViSQOL ↑ | Recon. CER (%) ↓ | TTS WER/CER (%) ↓ | TTS UTMOSv2 ↑ |
|
| 168 |
+
|---|---|---|---|---|---|---|---|---|---|
|
| 169 |
+
| LibriSpeech test-clean (EN) | LiquidCodec BF16 | 4.442 | 2.640 | 0.120 | 0.785 | 4.319 | 0.89 | 3.15 | 3.31 |
|
| 170 |
+
| | LiquidCodec Q4_0 | 4.445 | 2.577 | 0.121 | 0.781 | 4.292 | 0.89 | 3.18 | 3.22 |
|
| 171 |
+
| | X-Codec 2 | 4.436 | 2.434 | 0.123 | 0.754 | 4.119 | 0.88 | 4.19 | 3.27 |
|
| 172 |
+
| | NanoCodec | 4.486 | 2.942 | 0.122 | 0.836 | 4.219 | 0.76 | 3.32 | 3.19 |
|
| 173 |
+
| | Mimi (12 cb) | 4.364 | 2.615 | 0.160 | 0.714 | 4.031 | 0.87 | 6.27 | 2.89 |
|
| 174 |
+
| Seed-TTS (EN) | LiquidCodec BF16 | 4.463 | 2.326 | 0.130 | 0.751 | 4.369 | 0.86 | 3.50 | 3.39 |
|
| 175 |
+
| | LiquidCodec Q4_0 | 4.460 | 2.271 | 0.131 | 0.744 | 4.341 | 0.86 | 3.54 | 3.31 |
|
| 176 |
+
| | X-Codec 2 | 4.436 | 2.099 | 0.140 | 0.745 | 4.105 | 0.86 | 5.65 | 3.24 |
|
| 177 |
+
| | NanoCodec | 4.467 | 2.599 | 0.136 | 0.799 | 4.304 | 0.73 | 3.95 | 3.29 |
|
| 178 |
+
| | Mimi (12 cb) | 4.347 | 2.396 | 0.163 | 0.722 | 4.124 | 1.01 | 6.15 | 2.98 |
|
| 179 |
+
| Seed-TTS (ZH) | LiquidCodec BF16 | 4.407 | 2.429 | 0.118 | 0.776 | 4.313 | 6.93 | 1.19 | 3.01 |
|
| 180 |
+
| | LiquidCodec Q4_0 | 4.393 | 2.375 | 0.119 | 0.768 | 4.285 | 7.04 | 1.20 | 2.92 |
|
| 181 |
+
| | X-Codec 2 | 4.387 | 2.237 | 0.126 | 0.823 | 4.064 | 7.30 | 1.44 | 2.85 |
|
| 182 |
+
| | NanoCodec | 4.407 | 2.606 | 0.122 | 0.811 | 4.189 | 6.41 | 1.27 | 2.88 |
|
| 183 |
+
| | Mimi (12 cb) | 4.198 | 2.401 | 0.152 | 0.702 | 3.992 | 7.59 | 2.12 | 2.53 |
|
| 184 |
+
| FLEURS (DE) | LiquidCodec BF16 | 4.237 | 2.434 | 0.120 | 0.807 | 4.207 | 3.65 | 5.30 | 3.20 |
|
| 185 |
+
| | LiquidCodec Q4_0 | 4.224 | 2.391 | 0.121 | 0.818 | 4.162 | 3.67 | 5.38 | 3.12 |
|
| 186 |
+
| | X-Codec 2 | 4.256 | 2.085 | 0.125 | 0.661 | 3.942 | 4.39 | 5.44 | 3.08 |
|
| 187 |
+
| | NanoCodec | 4.202 | 2.597 | 0.129 | 0.822 | 4.009 | 3.74 | 5.36 | 3.01 |
|
| 188 |
+
| | Mimi (12 cb) | 4.087 | 2.393 | 0.170 | 0.698 | 3.627 | 5.36 | 6.14 | 2.78 |
|
| 189 |
+
| FLEURS (FR) | LiquidCodec BF16 | 4.301 | 2.315 | 0.127 | 0.825 | 4.242 | 5.27 | 6.91 | 3.12 |
|
| 190 |
+
| | LiquidCodec Q4_0 | 4.302 | 2.262 | 0.128 | 0.820 | 4.212 | 5.27 | 5.26 | 3.04 |
|
| 191 |
+
| | X-Codec 2 | 4.317 | 2.019 | 0.134 | 0.606 | 3.914 | 6.84 | 6.37 | 3.01 |
|
| 192 |
+
| | NanoCodec | 4.305 | 2.500 | 0.137 | 0.824 | 4.072 | 5.06 | 7.72 | 3.02 |
|
| 193 |
+
| | Mimi (12 cb) | 3.876 | 2.207 | 0.192 | 0.710 | 3.758 | 15.95 | 8.73 | 2.67 |
|
| 194 |
+
| FLEURS (ZH) | LiquidCodec BF16 | 4.343 | 2.479 | 0.121 | 0.794 | 4.273 | 7.60 | 8.82 | 3.02 |
|
| 195 |
+
| | LiquidCodec Q4_0 | 4.339 | 2.436 | 0.123 | 0.782 | 4.249 | 7.56 | 7.84 | 2.94 |
|
| 196 |
+
| | X-Codec 2 | 4.335 | 2.307 | 0.127 | 0.559 | 4.047 | 8.24 | 7.86 | 2.86 |
|
| 197 |
+
| | NanoCodec | 4.334 | 2.679 | 0.133 | 0.845 | 4.164 | 6.68 | 7.45 | 2.91 |
|
| 198 |
+
| | Mimi (12 cb) | 3.916 | 2.179 | 0.191 | 0.616 | 3.837 | 9.51 | 8.35 | 2.53 |
|
| 199 |
+
| FLEURS (JA) | LiquidCodec BF16 | 4.347 | 2.233 | 0.122 | 0.766 | 4.162 | 6.80 | 7.29 | 3.05 |
|
| 200 |
+
| | LiquidCodec Q4_0 | 4.327 | 2.188 | 0.123 | 0.756 | 4.120 | 6.83 | 7.32 | 2.98 |
|
| 201 |
+
| | X-Codec 2 | 4.354 | 2.045 | 0.130 | 0.497 | 3.805 | 7.02 | 5.40 | 2.94 |
|
| 202 |
+
| | NanoCodec | 4.378 | 2.415 | 0.129 | 0.695 | 4.089 | 6.02 | 5.58 | 2.84 |
|
| 203 |
+
| | Mimi (12 cb) | 4.332 | 2.232 | 0.166 | 0.658 | 3.982 | 12.13 | 6.27 | 2.49 |
|
| 204 |
+
| FLEURS (KO) | LiquidCodec BF16 | 4.217 | 2.637 | 0.117 | 0.803 | 4.176 | 4.45 | 8.36 | 2.97 |
|
| 205 |
+
| | LiquidCodec Q4_0 | 4.214 | 2.599 | 0.118 | 0.800 | 4.146 | 4.30 | 6.71 | 2.88 |
|
| 206 |
+
| | X-Codec 2 | 4.214 | 2.299 | 0.128 | 0.591 | 3.870 | 5.08 | 6.22 | 2.87 |
|
| 207 |
+
| | NanoCodec | 4.212 | 2.589 | 0.137 | 0.769 | 4.018 | 3.41 | 5.96 | 2.86 |
|
| 208 |
+
| | Mimi (12 cb) | 3.843 | 2.165 | 0.178 | 0.543 | 3.530 | 9.37 | 9.81 | 2.54 |
|
| 209 |
+
|
| 210 |
+
<details>
|
| 211 |
+
<summary><small>Evaluation details</small></summary>
|
| 212 |
+
|
| 213 |
+
<p><small>
|
| 214 |
+
<strong>Reconstruction</strong>: each recording is encoded into tokens and decoded back into speech; the reconstruction is compared against the original. CER is measured on the reconstructed audio.<br>
|
| 215 |
+
<strong>Zero-shot TTS</strong>: for each tokenizer, a TTS model is trained with identical data (101k hours across six languages), architecture and compute. The backbone is initialized from LFM3-1.7B, with a six-layer, 1,024-dim autoregressive depth decoder predicting the codebooks within each frame. Each model is trained with identical batches in identical order. Only the token-specific embeddings and output heads change. At evaluation, models receive text without a reference speech prompt; scores aggregate three generation seeds.<br>
|
| 216 |
+
<strong>Backbone check</strong>: repeating the TTS experiment with a Qwen3-1.7B backbone gives consistent results.<br>
|
| 217 |
+
<strong>Q4_0</strong>: decoder quantized to 4 bits after decoder-only quantization-aware training.
|
| 218 |
+
</small></p>
|
| 219 |
+
|
| 220 |
+
</details>
|
| 221 |
+
|
| 222 |
+
## Architecture
|
| 223 |
+
<hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
|
| 224 |
+
|
| 225 |
+
**Symmetric LFM3 encoder and decoder:** both are built from blocks of the LFM3 architecture, which is optimized for performance on the edge. The encoder mirrors the decoder exactly and is fully causal, so tokenization is as lightweight as detokenization. This suits real-time, low-latency streaming.
|
| 226 |
+
|
| 227 |
+
**Spectral processing:** the decoder predicts spectral coefficients directly. This avoids progressive upsampling with learned transposed convolutions, a costly step in similar codecs.
|
| 228 |
+
|
| 229 |
+
**Grouped finite scalar quantization:** FSQ rounds each dimension of a feature vector to one of a fixed number of values. GFSQ splits the features into groups that are quantized independently, and each group yields one token. Without RVQ's chain of dependencies between codebooks, multiple audio tokens can be predicted in parallel with MaskGIT-based methods.
|
| 230 |
+
|
| 231 |
+
## Citation
|
| 232 |
+
<hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
|
| 233 |
+
|
| 234 |
+
```bibtex
|
| 235 |
+
@article{liquidAI2026liquidcodec,
|
| 236 |
+
author = {Liquid AI},
|
| 237 |
+
title = {LiquidCodec: High-Fidelity Audio Tokenization Optimized for Speed},
|
| 238 |
+
journal = {Liquid AI Blog},
|
| 239 |
+
year = {2026},
|
| 240 |
+
note = {www.liquid.ai/blog/liquidcodec},
|
| 241 |
+
}
|
| 242 |
+
```
|
| 243 |
+
|
| 244 |
+
For questions and feedback, join [Discord](https://discord.com/invite/liquid-ai). For custom deployment and fine-tuning support, [contact Liquid AI](https://www.liquid.ai/connect?utm_source=huggingface&utm_medium=referral&utm_campaign=huggingface).
|
config.json
CHANGED
|
@@ -1,102 +1,45 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"architectures": [
|
| 3 |
"LiquidCodecModel"
|
| 4 |
],
|
| 5 |
-
"
|
| 6 |
-
|
| 7 |
-
"attention_dropout": 0.0,
|
| 8 |
-
"conv_L_cache": 7,
|
| 9 |
-
"conv_bias": false,
|
| 10 |
-
"frame_stack": 6,
|
| 11 |
-
"head_dim": 64,
|
| 12 |
-
"hidden_size": 512,
|
| 13 |
-
"hop_length": 320,
|
| 14 |
-
"initializer_range": 0.02,
|
| 15 |
-
"intermediate_size": 2304,
|
| 16 |
-
"layer_types": [
|
| 17 |
-
"conv",
|
| 18 |
-
"conv",
|
| 19 |
-
"sliding_attention",
|
| 20 |
-
"conv",
|
| 21 |
-
"sliding_attention",
|
| 22 |
-
"conv",
|
| 23 |
-
"sliding_attention",
|
| 24 |
-
"conv"
|
| 25 |
-
],
|
| 26 |
-
"max_position_embeddings": 131072,
|
| 27 |
-
"model_type": "liquid_codec_decoder",
|
| 28 |
-
"n_fft": 1280,
|
| 29 |
-
"norm_eps": 1e-05,
|
| 30 |
-
"num_attention_heads": 8,
|
| 31 |
-
"num_hidden_layers": 8,
|
| 32 |
-
"num_key_value_heads": 2,
|
| 33 |
-
"quantization_dim": 256,
|
| 34 |
-
"rope_parameters": {
|
| 35 |
-
"sliding_attention": {
|
| 36 |
-
"partial_rotary_factor": 1.0,
|
| 37 |
-
"rope_theta": 1000000.0,
|
| 38 |
-
"rope_type": "default"
|
| 39 |
-
}
|
| 40 |
-
},
|
| 41 |
-
"sliding_window": 30,
|
| 42 |
-
"use_attention_gate": true,
|
| 43 |
-
"use_cache": false,
|
| 44 |
-
"use_sandwich_norm": true,
|
| 45 |
-
"win_length": 1280
|
| 46 |
-
},
|
| 47 |
"dtype": "bfloat16",
|
| 48 |
-
"
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
"
|
| 52 |
-
"
|
| 53 |
-
"embedder_hidden_act": "silu",
|
| 54 |
-
"embedder_hidden_size": 256,
|
| 55 |
-
"frame_stack": 6,
|
| 56 |
-
"head_dim": 64,
|
| 57 |
-
"hidden_size": 512,
|
| 58 |
-
"initializer_range": 0.02,
|
| 59 |
-
"intermediate_size": 2304,
|
| 60 |
-
"layer_types": [
|
| 61 |
-
"conv",
|
| 62 |
-
"conv",
|
| 63 |
-
"full_attention",
|
| 64 |
-
"conv",
|
| 65 |
-
"full_attention",
|
| 66 |
-
"conv",
|
| 67 |
-
"full_attention",
|
| 68 |
-
"conv"
|
| 69 |
-
],
|
| 70 |
-
"max_position_embeddings": 131072,
|
| 71 |
-
"model_type": "liquid_codec_encoder",
|
| 72 |
-
"n_fft": 1280,
|
| 73 |
-
"norm_eps": 1e-05,
|
| 74 |
-
"num_attention_heads": 8,
|
| 75 |
-
"num_hidden_layers": 8,
|
| 76 |
-
"num_key_value_heads": 2,
|
| 77 |
-
"quantization_dim": 256,
|
| 78 |
-
"rope_parameters": {
|
| 79 |
-
"full_attention": {
|
| 80 |
-
"partial_rotary_factor": 1.0,
|
| 81 |
-
"rope_theta": 1000000.0,
|
| 82 |
-
"rope_type": "default"
|
| 83 |
-
}
|
| 84 |
-
},
|
| 85 |
-
"sliding_window": null,
|
| 86 |
-
"use_attention_gate": true,
|
| 87 |
-
"use_cache": false,
|
| 88 |
-
"use_sandwich_norm": true
|
| 89 |
},
|
| 90 |
-
"
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
8,
|
| 96 |
7,
|
| 97 |
6,
|
| 98 |
6
|
| 99 |
],
|
|
|
|
|
|
|
| 100 |
"sampling_rate": 24000,
|
| 101 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"transformers_version": "5.19.0.dev0",
|
| 3 |
"architectures": [
|
| 4 |
"LiquidCodecModel"
|
| 5 |
],
|
| 6 |
+
"output_hidden_states": false,
|
| 7 |
+
"return_dict": true,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
"dtype": "bfloat16",
|
| 9 |
+
"chunk_size_feed_forward": 0,
|
| 10 |
+
"is_encoder_decoder": false,
|
| 11 |
+
"id2label": {
|
| 12 |
+
"0": "LABEL_0",
|
| 13 |
+
"1": "LABEL_1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
},
|
| 15 |
+
"label2id": {
|
| 16 |
+
"LABEL_0": 0,
|
| 17 |
+
"LABEL_1": 1
|
| 18 |
+
},
|
| 19 |
+
"problem_type": null,
|
| 20 |
+
"encoder_dim": 512,
|
| 21 |
+
"encoder_conv_dim": 256,
|
| 22 |
+
"stft_n_fft": 1280,
|
| 23 |
+
"stft_hop_length": 320,
|
| 24 |
+
"stft_win_length": 1280,
|
| 25 |
+
"stft_frame_stack": 6,
|
| 26 |
+
"quantizer_dim": 256,
|
| 27 |
+
"fsq_levels": [
|
| 28 |
8,
|
| 29 |
7,
|
| 30 |
6,
|
| 31 |
6
|
| 32 |
],
|
| 33 |
+
"n_q": 12,
|
| 34 |
+
"decoder_window": 30,
|
| 35 |
"sampling_rate": 24000,
|
| 36 |
+
"initializer_range": 0.02,
|
| 37 |
+
"model_type": "liquid_codec",
|
| 38 |
+
"output_attentions": false,
|
| 39 |
+
"sample_rate": 24000,
|
| 40 |
+
"frame_samples": 1920,
|
| 41 |
+
"auto_map": {
|
| 42 |
+
"AutoConfig": "configuration_liquid_codec.LiquidCodecConfig",
|
| 43 |
+
"AutoModel": "modeling_liquid_codec.LiquidCodecModel"
|
| 44 |
+
}
|
| 45 |
}
|
configuration_lfm3.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 2 |
+
# This file was automatically generated from src/transformers/models/lfm3/modular_lfm3.py.
|
| 3 |
+
# Do NOT edit this file manually as any edits will be overwritten by the generation of
|
| 4 |
+
# the file from the modular. If any change should be done, please apply the change to the
|
| 5 |
+
# modular_lfm3.py file directly. One of our CI enforces this.
|
| 6 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 7 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 8 |
+
#
|
| 9 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 10 |
+
# you may not use this file except in compliance with the License.
|
| 11 |
+
# You may obtain a copy of the License at
|
| 12 |
+
#
|
| 13 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 14 |
+
#
|
| 15 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 16 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 17 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 18 |
+
# See the License for the specific language governing permissions and
|
| 19 |
+
# limitations under the License.
|
| 20 |
+
from huggingface_hub.dataclasses import strict
|
| 21 |
+
|
| 22 |
+
from transformers.configuration_utils import PreTrainedConfig
|
| 23 |
+
from transformers.utils import auto_docstring
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@auto_docstring(checkpoint="LiquidAI/LFM3-1.7B")
|
| 27 |
+
@strict
|
| 28 |
+
class Lfm3Config(PreTrainedConfig):
|
| 29 |
+
r"""
|
| 30 |
+
conv_bias (`bool`, *optional*, defaults to `False`):
|
| 31 |
+
Whether to use bias in the conv layers.
|
| 32 |
+
conv_L_cache (`int`, *optional*, defaults to 7):
|
| 33 |
+
L_cache dim in the conv layers.
|
| 34 |
+
use_sandwich_norm (`bool`, *optional*, defaults to `True`):
|
| 35 |
+
Whether to apply an RMSNorm to the operator and feed-forward outputs before they enter the residual stream.
|
| 36 |
+
use_attention_gate (`bool`, *optional*, defaults to `True`):
|
| 37 |
+
Whether the attention layers scale each head's output by a sigmoid gate computed from the layer input.
|
| 38 |
+
|
| 39 |
+
```python
|
| 40 |
+
>>> from transformers import Lfm3Model, Lfm3Config
|
| 41 |
+
|
| 42 |
+
>>> # Initializing a LFM3 model
|
| 43 |
+
>>> configuration = Lfm3Config()
|
| 44 |
+
|
| 45 |
+
>>> # Initializing a model from the LFM3-1.7B style configuration
|
| 46 |
+
>>> model = Lfm3Model(configuration)
|
| 47 |
+
|
| 48 |
+
>>> # Accessing the model configuration
|
| 49 |
+
>>> configuration = model.config
|
| 50 |
+
```
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
model_type = "lfm3"
|
| 54 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 55 |
+
default_theta = 1000000.0
|
| 56 |
+
|
| 57 |
+
vocab_size: int = 128_000
|
| 58 |
+
hidden_size: int = 2048
|
| 59 |
+
intermediate_size: int = 7680
|
| 60 |
+
num_hidden_layers: int = 24
|
| 61 |
+
num_attention_heads: int = 16
|
| 62 |
+
num_key_value_heads: int = 4
|
| 63 |
+
max_position_embeddings: int = 128_000
|
| 64 |
+
initializer_range: float = 0.02
|
| 65 |
+
norm_eps: float = 0.00001
|
| 66 |
+
use_cache: bool = True
|
| 67 |
+
pad_token_id: int | None = 124_893
|
| 68 |
+
bos_token_id: int | None = 124_894
|
| 69 |
+
eos_token_id: int | list[int] | None = 124_900
|
| 70 |
+
tie_word_embeddings: bool = True
|
| 71 |
+
rope_parameters: dict | None = None
|
| 72 |
+
conv_bias: bool = False
|
| 73 |
+
conv_L_cache: int = 7
|
| 74 |
+
layer_types: list[str] | None = None
|
| 75 |
+
attribute_map = {"rms_norm_eps": "norm_eps"}
|
| 76 |
+
_rope_type_labels = ("full_attention", "sliding_attention")
|
| 77 |
+
head_dim: int = 128
|
| 78 |
+
sliding_window: int | None = 256
|
| 79 |
+
use_sandwich_norm: bool = True
|
| 80 |
+
use_attention_gate: bool = True
|
| 81 |
+
attention_bias: bool = False
|
| 82 |
+
attention_dropout: float = 0.0
|
| 83 |
+
|
| 84 |
+
def __post_init__(self, **kwargs):
|
| 85 |
+
if self.rope_parameters is None:
|
| 86 |
+
self.rope_parameters = {
|
| 87 |
+
"sliding_attention": {"rope_type": "default", "rope_theta": 10_000.0},
|
| 88 |
+
"full_attention": {
|
| 89 |
+
"rope_type": "proportional",
|
| 90 |
+
"rope_theta": 1_000_000.0,
|
| 91 |
+
"partial_rotary_factor": 0.5,
|
| 92 |
+
},
|
| 93 |
+
}
|
| 94 |
+
if self.layer_types is None:
|
| 95 |
+
self.layer_types = [
|
| 96 |
+
"full_attention" if i % 4 == 2 else "sliding_attention" if i % 4 == 0 and i > 0 else "conv"
|
| 97 |
+
for i in range(self.num_hidden_layers)
|
| 98 |
+
]
|
| 99 |
+
|
| 100 |
+
super().__post_init__(**kwargs)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
__all__ = ["Lfm3Config"]
|
configuration_liquid_codec.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import math
|
| 16 |
+
|
| 17 |
+
from huggingface_hub.dataclasses import strict
|
| 18 |
+
|
| 19 |
+
from transformers.configuration_utils import PreTrainedConfig
|
| 20 |
+
from transformers.utils import auto_docstring
|
| 21 |
+
|
| 22 |
+
from .configuration_lfm3 import Lfm3Config
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@auto_docstring(checkpoint="LiquidAI/Liquid-Codec")
|
| 26 |
+
@strict
|
| 27 |
+
class LiquidCodecConfig(PreTrainedConfig):
|
| 28 |
+
r"""
|
| 29 |
+
encoder_dim (`int`, *optional*, defaults to 512):
|
| 30 |
+
The width of the encoder and decoder LFM3 models.
|
| 31 |
+
encoder_conv_dim (`int`, *optional*, defaults to 256):
|
| 32 |
+
The number of channels of the STFT front end.
|
| 33 |
+
stft_n_fft (`int`, *optional*, defaults to 1280):
|
| 34 |
+
The FFT size of the analysis and synthesis transforms.
|
| 35 |
+
stft_hop_length (`int`, *optional*, defaults to 320):
|
| 36 |
+
The hop of the analysis and synthesis transforms.
|
| 37 |
+
stft_win_length (`int`, *optional*, defaults to 1280):
|
| 38 |
+
The window length of the analysis and synthesis transforms.
|
| 39 |
+
stft_frame_stack (`int`, *optional*, defaults to 6):
|
| 40 |
+
The number of STFT frames per codec frame.
|
| 41 |
+
quantizer_dim (`int`, *optional*, defaults to 256):
|
| 42 |
+
The width of the latent handed to the quantizer.
|
| 43 |
+
fsq_levels (`list[int]`, *optional*, defaults to `[8, 7, 6, 6]`):
|
| 44 |
+
The number of levels of each finite-scalar-quantization dimension; their product is the codebook size.
|
| 45 |
+
n_q (`int`, *optional*, defaults to 12):
|
| 46 |
+
The number of codebooks.
|
| 47 |
+
decoder_window (`int`, *optional*, defaults to 30):
|
| 48 |
+
The sliding attention window of the decoder LFM3 model, in 75 Hz frames.
|
| 49 |
+
sampling_rate (`int`, *optional*, defaults to 24000):
|
| 50 |
+
The sampling rate of the audio.
|
| 51 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 52 |
+
The standard deviation of the truncated normal initializer of the weights.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
model_type = "liquid_codec"
|
| 56 |
+
|
| 57 |
+
encoder_dim: int = 512
|
| 58 |
+
encoder_conv_dim: int = 256
|
| 59 |
+
stft_n_fft: int = 1280
|
| 60 |
+
stft_hop_length: int = 320
|
| 61 |
+
stft_win_length: int = 1280
|
| 62 |
+
stft_frame_stack: int = 6
|
| 63 |
+
quantizer_dim: int = 256
|
| 64 |
+
fsq_levels: list[int] | None = None
|
| 65 |
+
n_q: int = 12
|
| 66 |
+
decoder_window: int = 30
|
| 67 |
+
sampling_rate: int = 24_000
|
| 68 |
+
initializer_range: float = 0.02
|
| 69 |
+
|
| 70 |
+
def __post_init__(self, **kwargs):
|
| 71 |
+
# the released configs nest the fields under `model` and dump the two LFM3 sub-configs
|
| 72 |
+
for key, value in kwargs.pop("model", {}).items():
|
| 73 |
+
if key != "n_output_samples":
|
| 74 |
+
setattr(self, key, value)
|
| 75 |
+
for key in ("encoder_lfm_config", "decoder_lfm_config", "n_output_samples"):
|
| 76 |
+
kwargs.pop(key, None)
|
| 77 |
+
if self.fsq_levels is None:
|
| 78 |
+
self.fsq_levels = [8, 7, 6, 6]
|
| 79 |
+
self.fsq_levels = list(self.fsq_levels)
|
| 80 |
+
super().__post_init__(**kwargs)
|
| 81 |
+
|
| 82 |
+
@property
|
| 83 |
+
def codebook_size(self) -> int:
|
| 84 |
+
return math.prod(self.fsq_levels)
|
| 85 |
+
|
| 86 |
+
@property
|
| 87 |
+
def frame_size(self) -> int:
|
| 88 |
+
return self.stft_hop_length * self.stft_frame_stack
|
| 89 |
+
|
| 90 |
+
def lfm_config(self, sliding_window: int | None = None) -> PreTrainedConfig:
|
| 91 |
+
attention_type = "full_attention" if sliding_window is None else "sliding_attention"
|
| 92 |
+
return Lfm3Config(
|
| 93 |
+
vocab_size=1,
|
| 94 |
+
pad_token_id=None,
|
| 95 |
+
bos_token_id=None,
|
| 96 |
+
eos_token_id=None,
|
| 97 |
+
hidden_size=self.encoder_dim,
|
| 98 |
+
intermediate_size=2304,
|
| 99 |
+
num_hidden_layers=8,
|
| 100 |
+
num_attention_heads=8,
|
| 101 |
+
num_key_value_heads=2,
|
| 102 |
+
head_dim=64,
|
| 103 |
+
max_position_embeddings=131_072,
|
| 104 |
+
norm_eps=1e-5,
|
| 105 |
+
conv_L_cache=7,
|
| 106 |
+
conv_bias=False,
|
| 107 |
+
use_sandwich_norm=True,
|
| 108 |
+
layer_types=[attention_type if i in (2, 4, 6) else "conv" for i in range(8)],
|
| 109 |
+
sliding_window=sliding_window,
|
| 110 |
+
rope_parameters={
|
| 111 |
+
attention_type: {"rope_type": "default", "rope_theta": 1_000_000.0, "partial_rotary_factor": 1.0}
|
| 112 |
+
},
|
| 113 |
+
use_cache=False,
|
| 114 |
+
tie_word_embeddings=False,
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
__all__ = ["LiquidCodecConfig"]
|
modeling_lfm3.py
ADDED
|
@@ -0,0 +1,658 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 2 |
+
# This file was automatically generated from src/transformers/models/lfm3/modular_lfm3.py.
|
| 3 |
+
# Do NOT edit this file manually as any edits will be overwritten by the generation of
|
| 4 |
+
# the file from the modular. If any change should be done, please apply the change to the
|
| 5 |
+
# modular_lfm3.py file directly. One of our CI enforces this.
|
| 6 |
+
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
| 7 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 8 |
+
#
|
| 9 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 10 |
+
# you may not use this file except in compliance with the License.
|
| 11 |
+
# You may obtain a copy of the License at
|
| 12 |
+
#
|
| 13 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 14 |
+
#
|
| 15 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 16 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 17 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 18 |
+
# See the License for the specific language governing permissions and
|
| 19 |
+
# limitations under the License.
|
| 20 |
+
from collections.abc import Callable
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
from torch import nn
|
| 25 |
+
|
| 26 |
+
from transformers import initialization as init
|
| 27 |
+
from transformers.activations import ACT2FN
|
| 28 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 29 |
+
from transformers.generation import GenerationMixin
|
| 30 |
+
from transformers.integrations import use_kernel_forward_from_hub
|
| 31 |
+
from transformers.integrations.accelerate import force_accelerate_hooks
|
| 32 |
+
from transformers.masking_utils import create_causal_mask, create_recurrent_attention_mask, create_sliding_window_causal_mask
|
| 33 |
+
from transformers.modeling_layers import GradientCheckpointingLayer
|
| 34 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
| 35 |
+
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
|
| 36 |
+
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel, slice_logits_to_keep
|
| 37 |
+
from transformers.processing_utils import Unpack
|
| 38 |
+
from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple
|
| 39 |
+
from transformers.utils.generic import merge_with_config_defaults
|
| 40 |
+
from transformers.utils.output_capturing import capture_outputs
|
| 41 |
+
from .configuration_lfm3 import Lfm3Config
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@use_kernel_forward_from_hub("RMSNorm")
|
| 45 |
+
class Lfm3RMSNorm(nn.Module):
|
| 46 |
+
def __init__(self, hidden_size, eps: float = 1e-6) -> None:
|
| 47 |
+
"""
|
| 48 |
+
Lfm3RMSNorm is equivalent to T5LayerNorm
|
| 49 |
+
"""
|
| 50 |
+
super().__init__()
|
| 51 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 52 |
+
self.variance_epsilon = eps
|
| 53 |
+
|
| 54 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 55 |
+
input_dtype = hidden_states.dtype
|
| 56 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 57 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 58 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 59 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 60 |
+
|
| 61 |
+
def extra_repr(self):
|
| 62 |
+
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class Lfm3RotaryEmbedding(nn.Module):
|
| 66 |
+
def __init__(self, config: Lfm3Config, device=None):
|
| 67 |
+
super().__init__()
|
| 68 |
+
self.max_seq_len_cached = config.max_position_embeddings
|
| 69 |
+
self.original_max_seq_len = config.max_position_embeddings
|
| 70 |
+
self.config = config
|
| 71 |
+
# diff -> only the attention layer types have a RoPE entry, `conv` layers have none
|
| 72 |
+
self.layer_types = [layer_type for layer_type in config._rope_type_labels if layer_type in config.layer_types]
|
| 73 |
+
self.rope_type = {}
|
| 74 |
+
for layer_type in self.layer_types:
|
| 75 |
+
self.rope_type[layer_type] = config.rope_parameters[layer_type]["rope_type"]
|
| 76 |
+
rope_init_fn: Callable = self.compute_default_rope_parameters
|
| 77 |
+
if self.rope_type[layer_type] != "default":
|
| 78 |
+
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type[layer_type]]
|
| 79 |
+
curr_inv_freq, curr_attention_scaling = rope_init_fn(config, device, layer_type=layer_type)
|
| 80 |
+
setattr(self, f"{layer_type}_inv_freq", nn.Buffer(curr_inv_freq, persistent=False))
|
| 81 |
+
setattr(self, f"{layer_type}_original_inv_freq", nn.Buffer(curr_inv_freq.clone(), persistent=False))
|
| 82 |
+
setattr(self, f"{layer_type}_attention_scaling", curr_attention_scaling)
|
| 83 |
+
|
| 84 |
+
@staticmethod
|
| 85 |
+
def compute_default_rope_parameters(
|
| 86 |
+
config: Lfm3Config, device=None, layer_type: str | None = None, **kwargs
|
| 87 |
+
) -> tuple[torch.Tensor, float]:
|
| 88 |
+
"""
|
| 89 |
+
Computes the inverse frequencies according to the original RoPE implementation
|
| 90 |
+
Args:
|
| 91 |
+
config ([`~transformers.PreTrainedConfig`]):
|
| 92 |
+
The model configuration.
|
| 93 |
+
layer_type (`str`):
|
| 94 |
+
The current layer type if the model has different RoPE parameters per type.
|
| 95 |
+
Should not be used unless `config.layer_types is not None`
|
| 96 |
+
|
| 97 |
+
Returns:
|
| 98 |
+
Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
|
| 99 |
+
post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
|
| 100 |
+
"""
|
| 101 |
+
# For backward compatibility standardize the `rope_parameters_dict` if it uses old format
|
| 102 |
+
base = config.rope_parameters[layer_type]["rope_theta"]
|
| 103 |
+
dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
|
| 104 |
+
|
| 105 |
+
attention_factor = 1.0 # Unused in this type of RoPE
|
| 106 |
+
# Compute the inverse frequencies
|
| 107 |
+
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
|
| 108 |
+
return inv_freq.to(device), attention_factor
|
| 109 |
+
|
| 110 |
+
@torch.no_grad()
|
| 111 |
+
@dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
|
| 112 |
+
def forward(self, x, position_ids, layer_type):
|
| 113 |
+
# diff -> returns cos/sin in fp32 without casting to `x.dtype`
|
| 114 |
+
inv_freq = getattr(self, f"{layer_type}_inv_freq")
|
| 115 |
+
attention_scaling = getattr(self, f"{layer_type}_attention_scaling")
|
| 116 |
+
|
| 117 |
+
freqs = position_ids[..., None].float() * inv_freq.to(device=x.device, dtype=torch.float)
|
| 118 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 119 |
+
cos = emb.cos() * attention_scaling
|
| 120 |
+
sin = emb.sin() * attention_scaling
|
| 121 |
+
|
| 122 |
+
return cos, sin
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class Lfm3MLP(nn.Module):
|
| 126 |
+
def __init__(self, config: Lfm3Config, intermediate_size: int | None = None):
|
| 127 |
+
super().__init__()
|
| 128 |
+
self.hidden_size = config.hidden_size
|
| 129 |
+
self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
|
| 130 |
+
self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 131 |
+
self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 132 |
+
self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
|
| 133 |
+
|
| 134 |
+
def forward(self, x):
|
| 135 |
+
return self.w2(F.silu(self.w1(x)) * self.w3(x))
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def rotate_half(x):
|
| 139 |
+
"""Rotates half the hidden dims of the input."""
|
| 140 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 141 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 142 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
@use_kernel_forward_from_hub("rotary_pos_emb")
|
| 146 |
+
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
|
| 147 |
+
"""Applies Rotary Position Embedding to the query and key tensors.
|
| 148 |
+
|
| 149 |
+
Args:
|
| 150 |
+
q (`torch.Tensor`): The query tensor.
|
| 151 |
+
k (`torch.Tensor`): The key tensor.
|
| 152 |
+
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
| 153 |
+
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
| 154 |
+
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
| 155 |
+
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
| 156 |
+
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
| 157 |
+
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
| 158 |
+
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
| 159 |
+
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
| 160 |
+
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
| 161 |
+
Returns:
|
| 162 |
+
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
| 163 |
+
"""
|
| 164 |
+
q_type, k_type = q.dtype, k.dtype
|
| 165 |
+
cos = cos.unsqueeze(unsqueeze_dim)
|
| 166 |
+
sin = sin.unsqueeze(unsqueeze_dim)
|
| 167 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 168 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 169 |
+
return q_embed.to(q_type), k_embed.to(k_type)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 173 |
+
"""
|
| 174 |
+
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
|
| 175 |
+
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
|
| 176 |
+
"""
|
| 177 |
+
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
| 178 |
+
if n_rep == 1:
|
| 179 |
+
return hidden_states
|
| 180 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
|
| 181 |
+
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def eager_attention_forward(
|
| 185 |
+
module: nn.Module,
|
| 186 |
+
query: torch.Tensor,
|
| 187 |
+
key: torch.Tensor,
|
| 188 |
+
value: torch.Tensor,
|
| 189 |
+
attention_mask: torch.Tensor | None,
|
| 190 |
+
scaling: float,
|
| 191 |
+
dropout: float = 0.0,
|
| 192 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 193 |
+
):
|
| 194 |
+
key_states = repeat_kv(key, module.num_key_value_groups)
|
| 195 |
+
value_states = repeat_kv(value, module.num_key_value_groups)
|
| 196 |
+
|
| 197 |
+
attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
|
| 198 |
+
if attention_mask is not None:
|
| 199 |
+
attn_weights = attn_weights + attention_mask
|
| 200 |
+
|
| 201 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
|
| 202 |
+
attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
|
| 203 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
| 204 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 205 |
+
|
| 206 |
+
return attn_output, attn_weights
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
class Lfm3Attention(nn.Module):
|
| 210 |
+
"""Qwen3 attention with RoPE in float32 and a sigmoid output gate per head."""
|
| 211 |
+
|
| 212 |
+
def __init__(self, config: Lfm3Config, layer_idx: int):
|
| 213 |
+
super().__init__()
|
| 214 |
+
self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None
|
| 215 |
+
self.config = config
|
| 216 |
+
self.layer_idx = layer_idx
|
| 217 |
+
self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
|
| 218 |
+
self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
|
| 219 |
+
self.scaling = self.head_dim**-0.5
|
| 220 |
+
self.attention_dropout = config.attention_dropout
|
| 221 |
+
self.is_causal = True
|
| 222 |
+
|
| 223 |
+
self.q_proj = nn.Linear(
|
| 224 |
+
config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
|
| 225 |
+
)
|
| 226 |
+
self.k_proj = nn.Linear(
|
| 227 |
+
config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
|
| 228 |
+
)
|
| 229 |
+
self.v_proj = nn.Linear(
|
| 230 |
+
config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
|
| 231 |
+
)
|
| 232 |
+
self.o_proj = nn.Linear(
|
| 233 |
+
config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
|
| 234 |
+
)
|
| 235 |
+
self.q_norm = Lfm3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # unlike olmo, only on the head dim!
|
| 236 |
+
self.k_norm = Lfm3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # thus post q_norm does not need reshape
|
| 237 |
+
self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None
|
| 238 |
+
self.g_proj = (
|
| 239 |
+
nn.Linear(config.hidden_size, config.num_attention_heads, bias=False)
|
| 240 |
+
if config.use_attention_gate
|
| 241 |
+
else None
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
def forward(
|
| 245 |
+
self,
|
| 246 |
+
hidden_states: torch.Tensor,
|
| 247 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 248 |
+
attention_mask: torch.Tensor | None,
|
| 249 |
+
past_key_values: Cache | None = None,
|
| 250 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 251 |
+
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
| 252 |
+
input_shape = hidden_states.shape[:-1]
|
| 253 |
+
hidden_shape = (*input_shape, -1, self.head_dim)
|
| 254 |
+
|
| 255 |
+
query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
|
| 256 |
+
key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
|
| 257 |
+
value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
| 258 |
+
|
| 259 |
+
cos, sin = position_embeddings
|
| 260 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
| 261 |
+
|
| 262 |
+
if past_key_values is not None:
|
| 263 |
+
key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
|
| 264 |
+
|
| 265 |
+
attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
|
| 266 |
+
self.config._attn_implementation, eager_attention_forward
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
attn_output, attn_weights = attention_interface(
|
| 270 |
+
self,
|
| 271 |
+
query_states,
|
| 272 |
+
key_states,
|
| 273 |
+
value_states,
|
| 274 |
+
attention_mask,
|
| 275 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 276 |
+
scaling=self.scaling,
|
| 277 |
+
sliding_window=self.sliding_window,
|
| 278 |
+
**kwargs,
|
| 279 |
+
)
|
| 280 |
+
if self.g_proj is not None:
|
| 281 |
+
attn_output = attn_output * torch.sigmoid(self.g_proj(hidden_states)).unsqueeze(-1)
|
| 282 |
+
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
|
| 283 |
+
attn_output = self.o_proj(attn_output)
|
| 284 |
+
return attn_output, attn_weights
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def apply_mask_to_padding_states(hidden_states, attention_mask):
|
| 288 |
+
"""
|
| 289 |
+
Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
|
| 290 |
+
"""
|
| 291 |
+
# NOTE: attention mask is a 2D boolean tensor
|
| 292 |
+
if attention_mask is not None:
|
| 293 |
+
dtype = hidden_states.dtype
|
| 294 |
+
hidden_states = (hidden_states * attention_mask[:, :, None]).to(dtype)
|
| 295 |
+
|
| 296 |
+
return hidden_states
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
# torch only: the hub causal_conv1d kernels serve widths 2-4, LFM3 uses 7
|
| 300 |
+
def causal_conv1d_update(
|
| 301 |
+
hidden_states: torch.Tensor,
|
| 302 |
+
conv_state: torch.Tensor,
|
| 303 |
+
weight: nn.Parameter,
|
| 304 |
+
bias: nn.Parameter | None = None,
|
| 305 |
+
activation: str | None = None,
|
| 306 |
+
):
|
| 307 |
+
_, hidden_size, seq_len = hidden_states.shape
|
| 308 |
+
state_len = conv_state.shape[-1]
|
| 309 |
+
|
| 310 |
+
hidden_states_new = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype)
|
| 311 |
+
conv_state.copy_(hidden_states_new[:, :, -state_len:])
|
| 312 |
+
out = F.conv1d(hidden_states_new, weight.unsqueeze(1), bias, padding=0, groups=hidden_size)
|
| 313 |
+
out = out[:, :, -seq_len:]
|
| 314 |
+
if activation is not None:
|
| 315 |
+
out = ACT2FN[activation](out)
|
| 316 |
+
return out.to(hidden_states.dtype)
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def causal_conv1d_fn(
|
| 320 |
+
hidden_states: torch.Tensor,
|
| 321 |
+
weight: nn.Parameter,
|
| 322 |
+
bias: nn.Parameter | None = None,
|
| 323 |
+
activation: str | None = None,
|
| 324 |
+
**kwargs,
|
| 325 |
+
):
|
| 326 |
+
_, hidden_size, seq_len = hidden_states.shape
|
| 327 |
+
padding = weight.shape[-1] - 1
|
| 328 |
+
|
| 329 |
+
out = F.conv1d(
|
| 330 |
+
hidden_states.to(weight.dtype),
|
| 331 |
+
weight=weight.unsqueeze(1),
|
| 332 |
+
bias=bias,
|
| 333 |
+
padding=padding,
|
| 334 |
+
groups=hidden_size,
|
| 335 |
+
)[:, :, :seq_len]
|
| 336 |
+
if activation is not None:
|
| 337 |
+
out = ACT2FN[activation](out)
|
| 338 |
+
return out.to(hidden_states.dtype)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
class Lfm3ShortConv(nn.Module):
|
| 342 |
+
def __init__(
|
| 343 |
+
self,
|
| 344 |
+
config: Lfm3Config,
|
| 345 |
+
layer_idx: int,
|
| 346 |
+
):
|
| 347 |
+
super().__init__()
|
| 348 |
+
self.config = config
|
| 349 |
+
self.layer_idx = layer_idx
|
| 350 |
+
self.conv_kernel_size = config.conv_L_cache
|
| 351 |
+
self.bias = config.conv_bias
|
| 352 |
+
|
| 353 |
+
self.conv = nn.Conv1d(
|
| 354 |
+
in_channels=config.hidden_size,
|
| 355 |
+
out_channels=config.hidden_size,
|
| 356 |
+
kernel_size=self.conv_kernel_size,
|
| 357 |
+
groups=config.hidden_size,
|
| 358 |
+
bias=self.bias,
|
| 359 |
+
padding=self.conv_kernel_size - 1,
|
| 360 |
+
)
|
| 361 |
+
self.in_proj = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=self.bias)
|
| 362 |
+
self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=self.bias)
|
| 363 |
+
|
| 364 |
+
self.layer_type = config.layer_types[layer_idx]
|
| 365 |
+
|
| 366 |
+
@force_accelerate_hooks("conv")
|
| 367 |
+
def forward(
|
| 368 |
+
self,
|
| 369 |
+
hidden_states: torch.Tensor,
|
| 370 |
+
past_key_values: Cache | None = None,
|
| 371 |
+
attention_mask: torch.Tensor | None = None,
|
| 372 |
+
**kwargs,
|
| 373 |
+
):
|
| 374 |
+
seq_len = hidden_states.shape[1]
|
| 375 |
+
|
| 376 |
+
hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
|
| 377 |
+
BCx = self.in_proj(hidden_states).transpose(-1, -2)
|
| 378 |
+
B, C, x = BCx.chunk(3, dim=-2)
|
| 379 |
+
hidden_states = B * x
|
| 380 |
+
|
| 381 |
+
if past_key_values is not None:
|
| 382 |
+
conv_state = past_key_values.get_conv_state(self.layer_idx, state_idx=0)
|
| 383 |
+
else:
|
| 384 |
+
conv_state = None
|
| 385 |
+
|
| 386 |
+
if conv_state is not None and seq_len == 1 and not past_key_values.is_recording_past(self.layer_idx):
|
| 387 |
+
# Single-token cached decode: the fused per-step kernel updates the conv state in-place.
|
| 388 |
+
hidden_states = causal_conv1d_update(
|
| 389 |
+
hidden_states, conv_state, self.conv.weight.squeeze(1), self.conv.bias
|
| 390 |
+
)
|
| 391 |
+
else:
|
| 392 |
+
if past_key_values is not None:
|
| 393 |
+
hidden_states = past_key_values.update_conv_state(
|
| 394 |
+
hidden_states, self.layer_idx, conv_kernel_size=self.conv_kernel_size
|
| 395 |
+
)
|
| 396 |
+
|
| 397 |
+
hidden_states = causal_conv1d_fn(hidden_states, self.conv.weight.squeeze(1), self.conv.bias, **kwargs)
|
| 398 |
+
|
| 399 |
+
# Drop the additional previous states
|
| 400 |
+
if past_key_values is not None:
|
| 401 |
+
hidden_states = hidden_states[:, :, -seq_len:]
|
| 402 |
+
|
| 403 |
+
y = C * hidden_states
|
| 404 |
+
y = y.transpose(-1, -2).contiguous()
|
| 405 |
+
y = self.out_proj(y)
|
| 406 |
+
return y
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
class Lfm3DecoderLayer(GradientCheckpointingLayer):
|
| 410 |
+
def __init__(self, config: Lfm3Config, layer_idx: int):
|
| 411 |
+
super().__init__()
|
| 412 |
+
self.is_attention_layer = config.layer_types[layer_idx] != "conv"
|
| 413 |
+
|
| 414 |
+
if self.is_attention_layer:
|
| 415 |
+
self.self_attn = Lfm3Attention(config, layer_idx)
|
| 416 |
+
else:
|
| 417 |
+
self.conv = Lfm3ShortConv(config, layer_idx)
|
| 418 |
+
self.feed_forward = Lfm3MLP(config)
|
| 419 |
+
self.operator_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
|
| 420 |
+
self.ffn_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
|
| 421 |
+
self.post_operator_norm = (
|
| 422 |
+
Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps) if config.use_sandwich_norm else nn.Identity()
|
| 423 |
+
)
|
| 424 |
+
self.post_ffn_norm = (
|
| 425 |
+
Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps) if config.use_sandwich_norm else nn.Identity()
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
def forward(
|
| 429 |
+
self,
|
| 430 |
+
hidden_states: torch.Tensor,
|
| 431 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
|
| 432 |
+
attention_mask: torch.Tensor | None = None,
|
| 433 |
+
position_ids: torch.LongTensor | None = None,
|
| 434 |
+
past_key_values: Cache | None = None,
|
| 435 |
+
**kwargs,
|
| 436 |
+
) -> torch.Tensor:
|
| 437 |
+
residual = hidden_states
|
| 438 |
+
if self.is_attention_layer:
|
| 439 |
+
hidden_states, _ = self.self_attn(
|
| 440 |
+
hidden_states=self.operator_norm(hidden_states),
|
| 441 |
+
position_embeddings=position_embeddings,
|
| 442 |
+
attention_mask=attention_mask,
|
| 443 |
+
position_ids=position_ids,
|
| 444 |
+
past_key_values=past_key_values,
|
| 445 |
+
**kwargs,
|
| 446 |
+
)
|
| 447 |
+
else:
|
| 448 |
+
hidden_states = self.conv(
|
| 449 |
+
hidden_states=self.operator_norm(hidden_states),
|
| 450 |
+
past_key_values=past_key_values,
|
| 451 |
+
attention_mask=attention_mask,
|
| 452 |
+
**kwargs,
|
| 453 |
+
)
|
| 454 |
+
hidden_states = self.post_operator_norm(hidden_states)
|
| 455 |
+
hidden_states = residual + hidden_states
|
| 456 |
+
|
| 457 |
+
residual = hidden_states
|
| 458 |
+
hidden_states = self.ffn_norm(hidden_states)
|
| 459 |
+
hidden_states = self.feed_forward(hidden_states)
|
| 460 |
+
hidden_states = self.post_ffn_norm(hidden_states)
|
| 461 |
+
hidden_states = residual + hidden_states
|
| 462 |
+
|
| 463 |
+
return hidden_states
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
@auto_docstring
|
| 467 |
+
class Lfm3PreTrainedModel(PreTrainedModel):
|
| 468 |
+
config: Lfm3Config
|
| 469 |
+
base_model_prefix = "model"
|
| 470 |
+
supports_gradient_checkpointing = True
|
| 471 |
+
_no_split_modules = ["Lfm3DecoderLayer"]
|
| 472 |
+
_skip_keys_device_placement = ["past_key_values"]
|
| 473 |
+
_supports_flash_attn = True
|
| 474 |
+
_supports_sdpa = True
|
| 475 |
+
_supports_flex_attn = True
|
| 476 |
+
|
| 477 |
+
_can_compile_fullgraph = True
|
| 478 |
+
_supports_attention_backend = True
|
| 479 |
+
_can_record_outputs = {
|
| 480 |
+
"hidden_states": Lfm3DecoderLayer,
|
| 481 |
+
"attentions": Lfm3Attention,
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
@torch.no_grad()
|
| 485 |
+
def _init_weights(self, module):
|
| 486 |
+
super()._init_weights(module)
|
| 487 |
+
if isinstance(module, Lfm3RotaryEmbedding):
|
| 488 |
+
for layer_type in module.layer_types:
|
| 489 |
+
rope_init_fn = module.compute_default_rope_parameters
|
| 490 |
+
if module.rope_type[layer_type] != "default":
|
| 491 |
+
rope_init_fn = ROPE_INIT_FUNCTIONS[module.rope_type[layer_type]]
|
| 492 |
+
curr_inv_freq, _ = rope_init_fn(module.config, layer_type=layer_type)
|
| 493 |
+
init.copy_(getattr(module, f"{layer_type}_inv_freq"), curr_inv_freq)
|
| 494 |
+
init.copy_(getattr(module, f"{layer_type}_original_inv_freq"), curr_inv_freq)
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
@auto_docstring
|
| 498 |
+
class Lfm3Model(Lfm3PreTrainedModel):
|
| 499 |
+
"""LFM2 model with a sliding-attention layer type; the mask and rotary embedding are built per layer type."""
|
| 500 |
+
|
| 501 |
+
def __init__(self, config: Lfm3Config):
|
| 502 |
+
super().__init__(config)
|
| 503 |
+
self.padding_idx = config.pad_token_id
|
| 504 |
+
self.vocab_size = config.vocab_size
|
| 505 |
+
|
| 506 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
| 507 |
+
self.layers = nn.ModuleList(
|
| 508 |
+
[Lfm3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
|
| 509 |
+
)
|
| 510 |
+
self.rotary_emb = Lfm3RotaryEmbedding(config=config)
|
| 511 |
+
self.gradient_checkpointing = False
|
| 512 |
+
self.embedding_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
|
| 513 |
+
|
| 514 |
+
# Initialize weights and apply final processing
|
| 515 |
+
self.post_init()
|
| 516 |
+
|
| 517 |
+
@merge_with_config_defaults
|
| 518 |
+
@capture_outputs
|
| 519 |
+
@auto_docstring
|
| 520 |
+
def forward(
|
| 521 |
+
self,
|
| 522 |
+
input_ids: torch.LongTensor | None = None,
|
| 523 |
+
attention_mask: torch.Tensor | None = None,
|
| 524 |
+
position_ids: torch.LongTensor | None = None,
|
| 525 |
+
past_key_values: Cache | None = None,
|
| 526 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 527 |
+
use_cache: bool | None = None,
|
| 528 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 529 |
+
) -> BaseModelOutputWithPast:
|
| 530 |
+
if (input_ids is None) ^ (inputs_embeds is not None):
|
| 531 |
+
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
|
| 532 |
+
|
| 533 |
+
if inputs_embeds is None:
|
| 534 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
| 535 |
+
|
| 536 |
+
if use_cache and past_key_values is None:
|
| 537 |
+
past_key_values = DynamicCache(config=self.config)
|
| 538 |
+
|
| 539 |
+
if position_ids is None:
|
| 540 |
+
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 541 |
+
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
|
| 542 |
+
position_ids = position_ids.unsqueeze(0)
|
| 543 |
+
|
| 544 |
+
if not isinstance(causal_mask_mapping := attention_mask, dict):
|
| 545 |
+
mask_kwargs = {
|
| 546 |
+
"config": self.config,
|
| 547 |
+
"inputs_embeds": inputs_embeds,
|
| 548 |
+
"attention_mask": attention_mask,
|
| 549 |
+
"past_key_values": past_key_values,
|
| 550 |
+
"position_ids": position_ids,
|
| 551 |
+
}
|
| 552 |
+
mask_creation_functions = {
|
| 553 |
+
"full_attention": lambda: create_causal_mask(**mask_kwargs),
|
| 554 |
+
"sliding_attention": lambda: create_sliding_window_causal_mask(**mask_kwargs),
|
| 555 |
+
"conv": lambda: create_recurrent_attention_mask(**mask_kwargs),
|
| 556 |
+
}
|
| 557 |
+
causal_mask_mapping = {}
|
| 558 |
+
for layer_type in set(self.config.layer_types):
|
| 559 |
+
causal_mask_mapping[layer_type] = mask_creation_functions[layer_type]()
|
| 560 |
+
|
| 561 |
+
hidden_states = inputs_embeds
|
| 562 |
+
position_embeddings = {
|
| 563 |
+
layer_type: self.rotary_emb(hidden_states, position_ids=position_ids, layer_type=layer_type)
|
| 564 |
+
for layer_type in self.rotary_emb.layer_types
|
| 565 |
+
}
|
| 566 |
+
|
| 567 |
+
for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
|
| 568 |
+
layer_type = self.config.layer_types[i]
|
| 569 |
+
hidden_states = decoder_layer(
|
| 570 |
+
hidden_states,
|
| 571 |
+
attention_mask=causal_mask_mapping[layer_type],
|
| 572 |
+
position_embeddings=position_embeddings.get(layer_type),
|
| 573 |
+
position_ids=position_ids,
|
| 574 |
+
past_key_values=past_key_values,
|
| 575 |
+
**kwargs,
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
hidden_states = self.embedding_norm(hidden_states)
|
| 579 |
+
|
| 580 |
+
return BaseModelOutputWithPast(
|
| 581 |
+
last_hidden_state=hidden_states,
|
| 582 |
+
past_key_values=past_key_values,
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
@auto_docstring
|
| 587 |
+
class Lfm3ForCausalLM(Lfm3PreTrainedModel, GenerationMixin):
|
| 588 |
+
_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
|
| 589 |
+
_tp_plan = {"lm_head": "colwise_gather_output"}
|
| 590 |
+
_pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
|
| 591 |
+
_fsdp_plan = {"lm_head": "keep_full_weight"}
|
| 592 |
+
|
| 593 |
+
def __init__(self, config: Lfm3Config):
|
| 594 |
+
super().__init__(config)
|
| 595 |
+
self.model = Lfm3Model(config)
|
| 596 |
+
self.vocab_size = config.vocab_size
|
| 597 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 598 |
+
|
| 599 |
+
# Initialize weights and apply final processing
|
| 600 |
+
self.post_init()
|
| 601 |
+
|
| 602 |
+
@can_return_tuple
|
| 603 |
+
@auto_docstring
|
| 604 |
+
def forward(
|
| 605 |
+
self,
|
| 606 |
+
input_ids: torch.LongTensor | None = None,
|
| 607 |
+
attention_mask: torch.Tensor | None = None,
|
| 608 |
+
position_ids: torch.LongTensor | None = None,
|
| 609 |
+
past_key_values: Cache | None = None,
|
| 610 |
+
inputs_embeds: torch.Tensor | None = None,
|
| 611 |
+
labels: torch.LongTensor | None = None,
|
| 612 |
+
use_cache: bool | None = None,
|
| 613 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 614 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 615 |
+
) -> CausalLMOutputWithPast:
|
| 616 |
+
r"""
|
| 617 |
+
Example:
|
| 618 |
+
|
| 619 |
+
```python
|
| 620 |
+
>>> from transformers import AutoTokenizer, Lfm3ForCausalLM
|
| 621 |
+
|
| 622 |
+
>>> model = Lfm3ForCausalLM.from_pretrained("LiquidAI/LFM3-1.7B")
|
| 623 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM3-1.7B")
|
| 624 |
+
|
| 625 |
+
>>> inputs = tokenizer.apply_chat_template(
|
| 626 |
+
... [{"role": "user", "content": "What is C. elegans?"}], add_generation_prompt=True, return_tensors="pt"
|
| 627 |
+
... )
|
| 628 |
+
>>> generate_ids = model.generate(**inputs, max_new_tokens=64)
|
| 629 |
+
>>> tokenizer.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True)
|
| 630 |
+
```"""
|
| 631 |
+
outputs: BaseModelOutputWithPast = self.model(
|
| 632 |
+
input_ids=input_ids,
|
| 633 |
+
attention_mask=attention_mask,
|
| 634 |
+
position_ids=position_ids,
|
| 635 |
+
past_key_values=past_key_values,
|
| 636 |
+
inputs_embeds=inputs_embeds,
|
| 637 |
+
use_cache=use_cache,
|
| 638 |
+
**kwargs,
|
| 639 |
+
)
|
| 640 |
+
|
| 641 |
+
hidden_states = outputs.last_hidden_state
|
| 642 |
+
hidden_states = slice_logits_to_keep(hidden_states, logits_to_keep)
|
| 643 |
+
logits = self.lm_head(hidden_states)
|
| 644 |
+
|
| 645 |
+
loss = None
|
| 646 |
+
if labels is not None:
|
| 647 |
+
loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
|
| 648 |
+
|
| 649 |
+
return CausalLMOutputWithPast(
|
| 650 |
+
loss=loss,
|
| 651 |
+
logits=logits,
|
| 652 |
+
past_key_values=outputs.past_key_values,
|
| 653 |
+
hidden_states=outputs.hidden_states,
|
| 654 |
+
attentions=outputs.attentions,
|
| 655 |
+
)
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
__all__ = ["Lfm3ForCausalLM", "Lfm3Model", "Lfm3PreTrainedModel"]
|
modeling_liquid_codec.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import math
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
import torch.nn.functional as F
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
from transformers import initialization as init
|
| 23 |
+
from transformers.cache_utils import DynamicCache
|
| 24 |
+
from transformers.modeling_utils import PreTrainedAudioTokenizerBase
|
| 25 |
+
from transformers.utils import ModelOutput, auto_docstring
|
| 26 |
+
|
| 27 |
+
from .configuration_liquid_codec import LiquidCodecConfig
|
| 28 |
+
from .modeling_lfm3 import Lfm3Model
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@auto_docstring
|
| 32 |
+
@dataclass
|
| 33 |
+
class LiquidCodecEncoderOutput(ModelOutput):
|
| 34 |
+
r"""
|
| 35 |
+
audio_codes (`torch.LongTensor` of shape `(batch_size, num_codebooks, num_frames)`):
|
| 36 |
+
The code of every codebook of every frame.
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
audio_codes: torch.LongTensor = None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@auto_docstring
|
| 43 |
+
@dataclass
|
| 44 |
+
class LiquidCodecDecoderOutput(ModelOutput):
|
| 45 |
+
r"""
|
| 46 |
+
audio_values (`torch.FloatTensor` of shape `(batch_size, num_samples)`):
|
| 47 |
+
The decoded 24 kHz audio in float32.
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
audio_values: torch.FloatTensor = None
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class LiquidCodecStreamingCache:
|
| 54 |
+
"""Carries the decoder state between chunks: the LFM3 cache and the unfinished overlap-add tail."""
|
| 55 |
+
|
| 56 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 57 |
+
self.past_key_values = DynamicCache(config=config.lfm_config(config.decoder_window))
|
| 58 |
+
self.audio: torch.Tensor | None = None
|
| 59 |
+
self.envelope: torch.Tensor | None = None
|
| 60 |
+
self.started = False
|
| 61 |
+
self.finished = False
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class LiquidCodecSTFT(nn.Module):
|
| 65 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 66 |
+
super().__init__()
|
| 67 |
+
self.n_fft = config.stft_n_fft
|
| 68 |
+
self.hop_length = config.stft_hop_length
|
| 69 |
+
self.win_length = config.stft_win_length
|
| 70 |
+
self.window = nn.Buffer(torch.hann_window(self.win_length, dtype=torch.float32), persistent=False)
|
| 71 |
+
self.num_features = 3 * (self.n_fft // 2 + 1)
|
| 72 |
+
|
| 73 |
+
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
| 74 |
+
total_pad = self.win_length - self.hop_length
|
| 75 |
+
audio = F.pad(audio.float(), (total_pad // 2, total_pad - total_pad // 2), mode="reflect")
|
| 76 |
+
spec = torch.stft(
|
| 77 |
+
audio,
|
| 78 |
+
n_fft=self.n_fft,
|
| 79 |
+
hop_length=self.hop_length,
|
| 80 |
+
win_length=self.win_length,
|
| 81 |
+
window=self.window.float(),
|
| 82 |
+
center=False,
|
| 83 |
+
return_complex=True,
|
| 84 |
+
)
|
| 85 |
+
magnitude = spec.abs()
|
| 86 |
+
phase_denominator = magnitude.clamp_min(1e-5)
|
| 87 |
+
return torch.cat((torch.log1p(magnitude), spec.real / phase_denominator, spec.imag / phase_denominator), dim=1)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class LiquidCodecFrontEnd(nn.Module):
|
| 91 |
+
def __init__(self, config: LiquidCodecConfig, num_features: int):
|
| 92 |
+
super().__init__()
|
| 93 |
+
self.frame_stack = config.stft_frame_stack
|
| 94 |
+
self.in_proj = nn.Conv1d(num_features, config.encoder_conv_dim, kernel_size=1)
|
| 95 |
+
self.conv = nn.Conv1d(config.encoder_conv_dim, config.encoder_conv_dim, kernel_size=3, padding=2)
|
| 96 |
+
self.out = nn.Conv1d(
|
| 97 |
+
config.encoder_conv_dim, config.encoder_dim, kernel_size=self.frame_stack, stride=self.frame_stack
|
| 98 |
+
)
|
| 99 |
+
self.act = nn.SiLU()
|
| 100 |
+
|
| 101 |
+
def forward(self, features: torch.Tensor) -> torch.Tensor:
|
| 102 |
+
hidden_states = self.act(self.in_proj(features.to(self.in_proj.weight.dtype)))
|
| 103 |
+
hidden_states = self.act(self.conv(hidden_states)[:, :, : hidden_states.shape[-1]])
|
| 104 |
+
return self.out(hidden_states)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class LiquidCodecLfm(nn.Module):
|
| 108 |
+
def __init__(self, config: LiquidCodecConfig, sliding_window: int | None = None):
|
| 109 |
+
super().__init__()
|
| 110 |
+
self.lfm = Lfm3Model(config.lfm_config(sliding_window))
|
| 111 |
+
self.lfm.register_module("embed_tokens", None)
|
| 112 |
+
# the projections run in the model dtype but the rotary tables stay in float32; a named hook keeps the
|
| 113 |
+
# module picklable
|
| 114 |
+
self.lfm.rotary_emb.register_forward_pre_hook(self._rotary_in_float32)
|
| 115 |
+
|
| 116 |
+
@staticmethod
|
| 117 |
+
def _rotary_in_float32(module, args):
|
| 118 |
+
return (args[0].float(), *args[1:])
|
| 119 |
+
|
| 120 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 121 |
+
return self.lfm(inputs_embeds=hidden_states, use_cache=False).last_hidden_state
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class LiquidCodecQuantizer(nn.Module):
|
| 125 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 126 |
+
super().__init__()
|
| 127 |
+
levels = list(config.fsq_levels)
|
| 128 |
+
self.num_codebooks = config.n_q
|
| 129 |
+
self.codebook_dim = len(levels)
|
| 130 |
+
self.levels = nn.Buffer(torch.tensor(levels, dtype=torch.int32), persistent=False)
|
| 131 |
+
self.basis = nn.Buffer(
|
| 132 |
+
torch.tensor([math.prod(levels[:index]) for index in range(self.codebook_dim)], dtype=torch.int32),
|
| 133 |
+
persistent=False,
|
| 134 |
+
)
|
| 135 |
+
effective_dim = self.codebook_dim * self.num_codebooks
|
| 136 |
+
if config.quantizer_dim == effective_dim:
|
| 137 |
+
self.project_in = nn.Identity()
|
| 138 |
+
self.project_out = nn.Identity()
|
| 139 |
+
else:
|
| 140 |
+
self.project_in = nn.Linear(config.quantizer_dim, effective_dim)
|
| 141 |
+
self.project_out = nn.Linear(effective_dim, config.quantizer_dim)
|
| 142 |
+
|
| 143 |
+
def _bound(self, latents: torch.Tensor, eps: float = 1e-3) -> torch.Tensor:
|
| 144 |
+
half_levels = (self.levels - 1) * (1 + eps) / 2
|
| 145 |
+
offset = torch.where(self.levels % 2 == 0, 0.5, 0.0)
|
| 146 |
+
shift = torch.atanh(offset / half_levels)
|
| 147 |
+
return torch.tanh(latents + shift) * half_levels - offset
|
| 148 |
+
|
| 149 |
+
def encode(self, latents: torch.Tensor) -> torch.Tensor:
|
| 150 |
+
latents = self.project_in(latents.movedim(1, -1).contiguous()).float()
|
| 151 |
+
latents = latents.reshape(*latents.shape[:-1], self.num_codebooks, self.codebook_dim)
|
| 152 |
+
half_width = self.levels // 2
|
| 153 |
+
codes = self._bound(latents).round() / half_width
|
| 154 |
+
indices = ((codes * half_width + half_width) * self.basis).sum(-1).round().long()
|
| 155 |
+
return indices.transpose(1, 2)
|
| 156 |
+
|
| 157 |
+
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
| 158 |
+
indices = codes.transpose(1, 2).unsqueeze(-1)
|
| 159 |
+
half_width = self.levels // 2
|
| 160 |
+
latents = ((indices // self.basis) % self.levels - half_width) / half_width
|
| 161 |
+
latents = latents.reshape(*latents.shape[:-2], -1)
|
| 162 |
+
if isinstance(self.project_out, nn.Linear):
|
| 163 |
+
latents = latents.to(self.project_out.weight.dtype)
|
| 164 |
+
return self.project_out(latents).movedim(-1, 1)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
class LiquidCodecISTFT(nn.Module):
|
| 168 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 169 |
+
super().__init__()
|
| 170 |
+
self.n_fft = config.stft_n_fft
|
| 171 |
+
self.hop_length = config.stft_hop_length
|
| 172 |
+
self.win_length = config.stft_win_length
|
| 173 |
+
self.window = nn.Buffer(torch.hann_window(self.win_length, dtype=torch.float32))
|
| 174 |
+
|
| 175 |
+
def forward(self, spec: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 176 |
+
num_frames = spec.shape[-1]
|
| 177 |
+
frames = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward") * self.window[None, :, None]
|
| 178 |
+
output_size = (num_frames - 1) * self.hop_length + self.win_length
|
| 179 |
+
fold = {"output_size": (1, output_size), "kernel_size": (1, self.win_length), "stride": (1, self.hop_length)}
|
| 180 |
+
audio = F.fold(frames, **fold)[:, 0, 0]
|
| 181 |
+
envelope = F.fold(self.window.square().expand(1, num_frames, -1).transpose(1, 2), **fold)[0, 0, 0]
|
| 182 |
+
return audio, envelope
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
class LiquidCodecDecoder(nn.Module):
|
| 186 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 187 |
+
super().__init__()
|
| 188 |
+
self.frame_stack = config.stft_frame_stack
|
| 189 |
+
self.lfm = LiquidCodecLfm(config, sliding_window=config.decoder_window).lfm
|
| 190 |
+
self.lin = nn.Linear(config.encoder_dim, 2 * (config.stft_n_fft // 2 + 1))
|
| 191 |
+
self.istft = LiquidCodecISTFT(config)
|
| 192 |
+
|
| 193 |
+
def forward(
|
| 194 |
+
self, embeddings: torch.Tensor, past_key_values: DynamicCache | None = None
|
| 195 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 196 |
+
hidden_states = self.lfm(
|
| 197 |
+
inputs_embeds=F.interpolate(
|
| 198 |
+
embeddings.mT, embeddings.shape[1] * self.frame_stack, mode="nearest-exact"
|
| 199 |
+
).mT,
|
| 200 |
+
past_key_values=past_key_values,
|
| 201 |
+
use_cache=past_key_values is not None,
|
| 202 |
+
).last_hidden_state
|
| 203 |
+
log_magnitude, angle = torch.chunk(self.lin(hidden_states).mT.contiguous(), 2, 1)
|
| 204 |
+
return self.istft(torch.polar(log_magnitude.float().exp(), angle.float()))
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
@auto_docstring
|
| 208 |
+
class LiquidCodecModel(PreTrainedAudioTokenizerBase):
|
| 209 |
+
config: LiquidCodecConfig
|
| 210 |
+
base_model_prefix = "codec"
|
| 211 |
+
main_input_name = "input_values"
|
| 212 |
+
_keep_in_fp32_modules_strict = ["stft", "istft"]
|
| 213 |
+
_supports_flash_attn = False
|
| 214 |
+
_supports_sdpa = True
|
| 215 |
+
|
| 216 |
+
@classmethod
|
| 217 |
+
def from_pretrained(cls, *args, **kwargs):
|
| 218 |
+
# Remote models do not inherit Transformers' built-in checkpoint key conversions.
|
| 219 |
+
kwargs.setdefault(
|
| 220 |
+
"key_mapping",
|
| 221 |
+
{
|
| 222 |
+
r"^_frontend\.": "frontend.",
|
| 223 |
+
r"^quantizer\.fsq\.": "quantizer.",
|
| 224 |
+
r"\.self_attn\.gate_proj\.": ".self_attn.g_proj.",
|
| 225 |
+
},
|
| 226 |
+
)
|
| 227 |
+
return super().from_pretrained(*args, **kwargs)
|
| 228 |
+
|
| 229 |
+
def __init__(self, config: LiquidCodecConfig):
|
| 230 |
+
super().__init__(config)
|
| 231 |
+
self.stft = LiquidCodecSTFT(config)
|
| 232 |
+
self.frontend = LiquidCodecFrontEnd(config, self.stft.num_features)
|
| 233 |
+
self.encoder = LiquidCodecLfm(config)
|
| 234 |
+
self.to_quantizer = nn.Linear(config.encoder_dim, config.quantizer_dim)
|
| 235 |
+
self.quantizer = LiquidCodecQuantizer(config)
|
| 236 |
+
self.decoder = LiquidCodecDecoder(config)
|
| 237 |
+
self.from_quantizer = nn.Linear(config.quantizer_dim, config.encoder_dim)
|
| 238 |
+
self.post_init()
|
| 239 |
+
|
| 240 |
+
@torch.no_grad()
|
| 241 |
+
def _init_weights(self, module):
|
| 242 |
+
super()._init_weights(module)
|
| 243 |
+
if isinstance(module, (LiquidCodecSTFT, LiquidCodecISTFT)):
|
| 244 |
+
init.copy_(module.window, torch.hann_window(module.win_length, dtype=torch.float32))
|
| 245 |
+
elif isinstance(module, LiquidCodecQuantizer):
|
| 246 |
+
levels = list(self.config.fsq_levels)
|
| 247 |
+
init.copy_(module.levels, torch.tensor(levels, dtype=torch.int32))
|
| 248 |
+
init.copy_(
|
| 249 |
+
module.basis,
|
| 250 |
+
torch.tensor([math.prod(levels[:index]) for index in range(len(levels))], dtype=torch.int32),
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
def encode(self, input_values: torch.FloatTensor) -> LiquidCodecEncoderOutput:
|
| 254 |
+
r"""
|
| 255 |
+
input_values (`torch.FloatTensor` of shape `(batch_size, num_samples)`):
|
| 256 |
+
24 kHz audio. The tail is zero-padded to a whole codec frame of 1920 samples.
|
| 257 |
+
"""
|
| 258 |
+
if input_values.ndim != 2 or input_values.shape[-1] == 0:
|
| 259 |
+
raise ValueError(
|
| 260 |
+
f"Expected non-empty audio of shape (batch_size, num_samples), got {tuple(input_values.shape)}"
|
| 261 |
+
)
|
| 262 |
+
frame_size = self.config.frame_size
|
| 263 |
+
num_samples = -(input_values.shape[-1] // -frame_size) * frame_size
|
| 264 |
+
input_values = F.pad(input_values, (0, num_samples - input_values.shape[-1]))
|
| 265 |
+
hidden_states = self.frontend(self.stft(input_values)).mT
|
| 266 |
+
latents = self.to_quantizer(self.encoder(hidden_states)).mT
|
| 267 |
+
return LiquidCodecEncoderOutput(audio_codes=self.quantizer.encode(latents))
|
| 268 |
+
|
| 269 |
+
def decode(
|
| 270 |
+
self,
|
| 271 |
+
audio_codes: torch.LongTensor,
|
| 272 |
+
streaming_cache: LiquidCodecStreamingCache | None = None,
|
| 273 |
+
finalize: bool = True,
|
| 274 |
+
) -> LiquidCodecDecoderOutput:
|
| 275 |
+
r"""
|
| 276 |
+
audio_codes (`torch.LongTensor` of shape `(batch_size, num_codebooks, num_frames)`):
|
| 277 |
+
The codes to decode; every frame becomes 1920 samples.
|
| 278 |
+
streaming_cache ([`LiquidCodecStreamingCache`], *optional*):
|
| 279 |
+
The state carried between consecutive chunks of one stream. The chunks decode to the same audio as the
|
| 280 |
+
concatenated codes.
|
| 281 |
+
finalize (`bool`, *optional*, defaults to `True`):
|
| 282 |
+
Whether this is the last chunk of the stream. Until then the trailing samples that later frames still
|
| 283 |
+
contribute to are held back in the cache.
|
| 284 |
+
"""
|
| 285 |
+
if audio_codes.ndim != 3 or audio_codes.shape[1] != self.config.n_q:
|
| 286 |
+
raise ValueError(
|
| 287 |
+
f"Expected codes of shape (batch_size, {self.config.n_q}, num_frames), got {tuple(audio_codes.shape)}"
|
| 288 |
+
)
|
| 289 |
+
if audio_codes.numel() and bool(((audio_codes < 0) | (audio_codes >= self.config.codebook_size)).any()):
|
| 290 |
+
raise ValueError(f"Codes must be in [0, {self.config.codebook_size})")
|
| 291 |
+
if streaming_cache is not None and streaming_cache.finished:
|
| 292 |
+
raise ValueError("This stream is finalized; start with a new cache")
|
| 293 |
+
trim = (self.config.stft_win_length - self.config.stft_hop_length) // 2
|
| 294 |
+
tail = 2 * trim
|
| 295 |
+
batch_size = audio_codes.shape[0]
|
| 296 |
+
if not audio_codes.shape[-1] and not (finalize and streaming_cache is not None and streaming_cache.started):
|
| 297 |
+
# no frames and no held-back tail to flush: the overlap-add state must stay untouched
|
| 298 |
+
if streaming_cache is not None and finalize:
|
| 299 |
+
streaming_cache.finished = True
|
| 300 |
+
return LiquidCodecDecoderOutput(audio_values=audio_codes.new_zeros((batch_size, 0), dtype=torch.float32))
|
| 301 |
+
if audio_codes.shape[-1]:
|
| 302 |
+
embeddings = self.from_quantizer(self.quantizer.decode(audio_codes).mT)
|
| 303 |
+
past_key_values = None if streaming_cache is None else streaming_cache.past_key_values
|
| 304 |
+
audio, envelope = self.decoder(embeddings, past_key_values)
|
| 305 |
+
else:
|
| 306 |
+
audio = self.decoder.istft.window.new_zeros(batch_size, tail)
|
| 307 |
+
envelope = self.decoder.istft.window.new_zeros(tail)
|
| 308 |
+
if streaming_cache is None:
|
| 309 |
+
audio, envelope = audio[:, trim:-trim], envelope[trim:-trim]
|
| 310 |
+
else:
|
| 311 |
+
if streaming_cache.audio is not None:
|
| 312 |
+
audio = torch.cat([audio[:, :tail] + streaming_cache.audio, audio[:, tail:]], dim=1)
|
| 313 |
+
envelope = torch.cat([envelope[:tail] + streaming_cache.envelope, envelope[tail:]])
|
| 314 |
+
if not streaming_cache.started:
|
| 315 |
+
audio, envelope = audio[:, trim:], envelope[trim:]
|
| 316 |
+
streaming_cache.started = True
|
| 317 |
+
if finalize:
|
| 318 |
+
audio, envelope = audio[:, :-trim], envelope[:-trim]
|
| 319 |
+
streaming_cache.audio, streaming_cache.envelope = None, None
|
| 320 |
+
streaming_cache.finished = True
|
| 321 |
+
else:
|
| 322 |
+
streaming_cache.audio, streaming_cache.envelope = audio[:, -tail:], envelope[-tail:]
|
| 323 |
+
audio, envelope = audio[:, :-tail], envelope[:-tail]
|
| 324 |
+
return LiquidCodecDecoderOutput(audio_values=(audio / envelope).float())
|
| 325 |
+
|
| 326 |
+
@auto_docstring
|
| 327 |
+
def forward(self, input_values: torch.FloatTensor, **kwargs) -> tuple[torch.FloatTensor, torch.LongTensor]:
|
| 328 |
+
r"""
|
| 329 |
+
input_values (`torch.FloatTensor` of shape `(batch_size, num_samples)`):
|
| 330 |
+
24 kHz audio.
|
| 331 |
+
|
| 332 |
+
Returns:
|
| 333 |
+
The reconstructed audio of the input length and the codes.
|
| 334 |
+
"""
|
| 335 |
+
audio_codes = self.encode(input_values).audio_codes
|
| 336 |
+
audio = self.decode(audio_codes).audio_values
|
| 337 |
+
return audio[..., : input_values.shape[-1]], audio_codes
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
__all__ = ["LiquidCodecModel", "LiquidCodecStreamingCache"]
|
preprocessor_config.json
DELETED
|
@@ -1,12 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"feature_extractor_type": "LiquidCodecFeatureExtractor",
|
| 3 |
-
"feature_size": 1923,
|
| 4 |
-
"frame_stack": 6,
|
| 5 |
-
"hop_length": 320,
|
| 6 |
-
"n_fft": 1280,
|
| 7 |
-
"padding_side": "right",
|
| 8 |
-
"padding_value": 0.0,
|
| 9 |
-
"return_attention_mask": true,
|
| 10 |
-
"sampling_rate": 24000,
|
| 11 |
-
"win_length": 1280
|
| 12 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|