Transformers-native config and feature extractor

#1
by eustlb HF Staff - opened
Files changed (3) hide show
  1. README.md +120 -0
  2. config.json +93 -1
  3. preprocessor_config.json +12 -0
README.md CHANGED
@@ -146,6 +146,126 @@ chunks.append(codec.decode(codes[..., :0], cache, finalize=True).audio_values)
146
  streamed = torch.cat(chunks, dim=-1)
147
  ```
148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  ## Performance
150
  <hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
151
 
 
146
  streamed = torch.cat(chunks, dim=-1)
147
  ```
148
 
149
+ ### Transformers usage 🤗
150
+
151
+ Liquid Codec is supported natively in 🤗 Transformers, without `trust_remote_code`. Install 🤗 Transformers from source:
152
+
153
+ ```sh
154
+ pip install git+https://github.com/huggingface/transformers
155
+ ```
156
+
157
+ <details>
158
+ <summary>Encoding and decoding</summary>
159
+
160
+ ```python
161
+ from transformers import AutoFeatureExtractor, AutoModel
162
+ from transformers.audio_utils import load_audio
163
+
164
+ model_id = "LiquidAI/Liquid-Codec"
165
+ model = AutoModel.from_pretrained(model_id, device_map="auto")
166
+ feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
167
+
168
+ url = "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/librispeech_mr_quilter.wav"
169
+ audio = load_audio(url, sampling_rate=feature_extractor.sampling_rate)
170
+ inputs = feature_extractor(
171
+ audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt", device=model.device
172
+ ).to(model.device, model.dtype)
173
+
174
+ audio_codes = model.encode(**inputs).audio_codes
175
+ print("Audio codes shape:", audio_codes.shape)
176
+ # Audio codes shape: torch.Size([1, 12, 74])
177
+ audio_values = model.decode(audio_codes).audio_values
178
+ print("Audio values shape:", audio_values.shape)
179
+ # Audio values shape: torch.Size([1, 142080])
180
+ ```
181
+
182
+ </details>
183
+
184
+ <details>
185
+ <summary>Batch processing</summary>
186
+
187
+ Clips of different lengths are padded to the longest one. Pass the masks along: `encode` returns the
188
+ `audio_codes_mask` of the valid frames of each clip, and `decode` the `audio_values_mask` of its valid samples.
189
+
190
+ ```python
191
+ from transformers import AutoFeatureExtractor, AutoModel
192
+ from transformers.audio_utils import load_audio
193
+
194
+ model_id = "LiquidAI/Liquid-Codec"
195
+ model = AutoModel.from_pretrained(model_id, device_map="auto")
196
+ feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
197
+
198
+ samples = "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/"
199
+ audios = [
200
+ load_audio(samples + name, sampling_rate=feature_extractor.sampling_rate)
201
+ for name in ("librispeech_mr_quilter.wav", "dude_where_is_my_car.wav")
202
+ ]
203
+ inputs = feature_extractor(
204
+ audio=audios, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt", device=model.device
205
+ ).to(model.device, model.dtype)
206
+
207
+ encoder_output = model.encode(**inputs)
208
+ print("Frames per clip:", encoder_output.audio_codes_mask.sum(-1).tolist())
209
+ # Frames per clip: [74, 814]
210
+
211
+ decoder_output = model.decode(encoder_output.audio_codes, encoder_output.audio_codes_mask)
212
+ print("Samples per clip:", decoder_output.audio_values_mask.sum(-1).tolist())
213
+ # Samples per clip: [142080, 1562880]
214
+
215
+ # drop the padding of each clip
216
+ audio_values = [
217
+ values[mask] for values, mask in zip(decoder_output.audio_values, decoder_output.audio_values_mask)
218
+ ]
219
+ ```
220
+
221
+ </details>
222
+
223
+ <details>
224
+ <summary>Streaming decoding</summary>
225
+
226
+ The decoder is causal, so audio can be synthesized chunk by chunk as the codes arrive.
227
+
228
+ ```python
229
+ import torch
230
+ from transformers import AutoFeatureExtractor, AutoModel
231
+ from transformers.audio_utils import load_audio
232
+
233
+ model_id = "LiquidAI/Liquid-Codec"
234
+ model = AutoModel.from_pretrained(model_id, device_map="auto")
235
+ feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
236
+
237
+ url = "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/librispeech_mr_quilter.wav"
238
+ audio = load_audio(url, sampling_rate=feature_extractor.sampling_rate)
239
+ inputs = feature_extractor(
240
+ audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt", device=model.device
241
+ ).to(model.device, model.dtype)
242
+ audio_codes = model.encode(**inputs).audio_codes
243
+
244
+ chunk_size = 4 # codec frames per chunk
245
+ decoder_cache, overlap_add_cache = None, None
246
+ chunks = []
247
+ for start in range(0, audio_codes.shape[-1], chunk_size):
248
+ outputs = model.decode(
249
+ audio_codes[..., start : start + chunk_size],
250
+ decoder_cache=decoder_cache,
251
+ overlap_add_cache=overlap_add_cache,
252
+ use_streaming=True,
253
+ )
254
+ decoder_cache, overlap_add_cache = outputs.decoder_cache, outputs.overlap_add_cache
255
+ chunks.append(outputs.audio_values) # play or send this chunk
256
+
257
+ # end of the stream
258
+ chunks.append(overlap_add_cache.flush().audio_values)
259
+
260
+ audio_values = torch.cat(chunks, dim=-1)
261
+ print("Audio values shape:", audio_values.shape)
262
+ # Audio values shape: torch.Size([1, 142080])
263
+ ```
264
+
265
+ </details>
266
+
267
+ See the [Transformers documentation](https://huggingface.co/docs/transformers/main/en/model_doc/liquid_codec) for more.
268
+
269
  ## Performance
270
  <hr style="margin: -0.7em 0 1.2em 0; border: 0; border-top: 1px solid rgba(127, 127, 127, 0.3);" />
271
 
config.json CHANGED
@@ -41,5 +41,97 @@
41
  "auto_map": {
42
  "AutoConfig": "configuration_liquid_codec.LiquidCodecConfig",
43
  "AutoModel": "modeling_liquid_codec.LiquidCodecModel"
44
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  }
 
41
  "auto_map": {
42
  "AutoConfig": "configuration_liquid_codec.LiquidCodecConfig",
43
  "AutoModel": "modeling_liquid_codec.LiquidCodecModel"
44
+ },
45
+ "decoder_config": {
46
+ "attention_bias": false,
47
+ "attention_dropout": 0.0,
48
+ "conv_L_cache": 7,
49
+ "conv_bias": false,
50
+ "frame_stack": 6,
51
+ "head_dim": 64,
52
+ "hidden_size": 512,
53
+ "hop_length": 320,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 2304,
56
+ "layer_types": [
57
+ "conv",
58
+ "conv",
59
+ "sliding_attention",
60
+ "conv",
61
+ "sliding_attention",
62
+ "conv",
63
+ "sliding_attention",
64
+ "conv"
65
+ ],
66
+ "max_position_embeddings": 131072,
67
+ "model_type": "liquid_codec_decoder",
68
+ "n_fft": 1280,
69
+ "norm_eps": 1e-05,
70
+ "num_attention_heads": 8,
71
+ "num_hidden_layers": 8,
72
+ "num_key_value_heads": 2,
73
+ "quantization_dim": 256,
74
+ "rope_parameters": {
75
+ "sliding_attention": {
76
+ "partial_rotary_factor": 1.0,
77
+ "rope_theta": 1000000.0,
78
+ "rope_type": "default"
79
+ }
80
+ },
81
+ "sliding_window": 30,
82
+ "use_attention_gate": true,
83
+ "use_cache": false,
84
+ "use_sandwich_norm": true,
85
+ "win_length": 1280
86
+ },
87
+ "encoder_config": {
88
+ "attention_bias": false,
89
+ "attention_dropout": 0.0,
90
+ "conv_L_cache": 7,
91
+ "conv_bias": false,
92
+ "embedder_hidden_act": "silu",
93
+ "embedder_hidden_size": 256,
94
+ "frame_stack": 6,
95
+ "head_dim": 64,
96
+ "hidden_size": 512,
97
+ "initializer_range": 0.02,
98
+ "intermediate_size": 2304,
99
+ "layer_types": [
100
+ "conv",
101
+ "conv",
102
+ "full_attention",
103
+ "conv",
104
+ "full_attention",
105
+ "conv",
106
+ "full_attention",
107
+ "conv"
108
+ ],
109
+ "max_position_embeddings": 131072,
110
+ "model_type": "liquid_codec_encoder",
111
+ "n_fft": 1280,
112
+ "norm_eps": 1e-05,
113
+ "num_attention_heads": 8,
114
+ "num_hidden_layers": 8,
115
+ "num_key_value_heads": 2,
116
+ "quantization_dim": 256,
117
+ "rope_parameters": {
118
+ "full_attention": {
119
+ "partial_rotary_factor": 1.0,
120
+ "rope_theta": 1000000.0,
121
+ "rope_type": "default"
122
+ }
123
+ },
124
+ "sliding_window": null,
125
+ "use_attention_gate": true,
126
+ "use_cache": false,
127
+ "use_sandwich_norm": true
128
+ },
129
+ "num_quantizers": 12,
130
+ "quantization_dim": 256,
131
+ "quantization_levels": [
132
+ 8,
133
+ 7,
134
+ 6,
135
+ 6
136
+ ]
137
  }
preprocessor_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "feature_extractor_type": "LiquidCodecFeatureExtractor",
3
+ "feature_size": 1923,
4
+ "frame_stack": 6,
5
+ "hop_length": 320,
6
+ "n_fft": 1280,
7
+ "padding_side": "right",
8
+ "padding_value": 0.0,
9
+ "return_attention_mask": true,
10
+ "sampling_rate": 24000,
11
+ "win_length": 1280
12
+ }