⚠️ DO NOT MERGE: feat(transformers): Add native Transformers weights and usage

README.md

@@ -47,6 +47,7 @@ tags:
   - pytorch
   - NeMo
   - hf-asr-leaderboard
+  - Transformers
 model-index:
   - name: canary-1b-v2
     results:
@@ -1896,6 +1897,10 @@ pip install -U nemo_toolkit['asr']
 ```
 The model is available for use in the NeMo toolkit [6], and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset.
 
+You can also run Canary with [Transformers](https://github.com/huggingface/transformers) 🤗 (more below).
+
+### 1) NeMo usage
+
 #### Automatically instantiate the model
 
 ```python
@@ -1955,6 +1960,110 @@ For translation task, please, refer to segment-level timestamps for getting intu
 > **Note:** If timestamps are not required for your work, you can reduce memory usage by restoring only the `.nemo` file without the auxiliary CTC model. To do this, extract the `.nemo` file, remove any *timestamps_asr_model* files, then repackage it into a new `.nemo` file.
 
 
+### 2) [Transformers](https://github.com/huggingface/transformers) 🤗 usage
+
+
+Until Canary is part of an official Transformers release, you can use it by installing from source.
+
+```bash
+pip install git+https://github.com/huggingface/transformers
+```
+
+<details>
+  <summary>➡️ Pipeline usage</summary>
+
+```python
+from transformers import pipeline
+
+pipe = pipeline("automatic-speech-recognition", model="nvidia/canary-1b-v2")
+out = pipe("https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3")
+print(out)
+```
+</details>
+
+<details>
+  <summary>➡️ Transcription</summary>
+
+```python
+from datasets import load_dataset, Audio
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
+
+model_id = "nvidia/canary-1b-v2"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, device_map="auto")
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+
+inputs = processor.apply_transcription_request(audio=ds[0]["audio"]["array"], source_language="en").to(model.device)
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+print(processor.decode(generated_ids, skip_special_tokens=True)[0])
+```
+</details>
+
+<details>
+  <summary>➡️ Translation</summary>
+
+```python
+inputs = processor.apply_transcription_request(
+    audio=ds[0]["audio"]["array"], source_language="en", target_language="de"
+).to(model.device)
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+print(processor.decode(generated_ids, skip_special_tokens=True)[0])
+```
+</details>
+
+<details>
+  <summary>➡️ Batch inference</summary>
+
+```python
+audios = [ds[0]["audio"]["array"], ds[1]["audio"]["array"]]
+inputs = processor.apply_transcription_request(
+    audio=audios, source_language="en", target_language=["en", "de"]
+).to(model.device)
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+for text in processor.decode(generated_ids, skip_special_tokens=True):
+    print(text)
+```
+</details>
+
+<details>
+  <summary>➡️ Training</summary>
+
+Put the target transcript in the assistant turn and pass `output_labels=True`. Padding positions are masked automatically.
+
+```python
+model.train()
+transcription = "mister Quilter is the apostle of the middle classes, and we are glad to welcome his gospel."
+
+conversation = [
+    [
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio", "audio": ds[0]["audio"]["array"]},
+                {"type": "text", "source_language": "en", "target_language": "en", "punctuation": True},
+            ],
+        },
+        {"role": "assistant", "content": transcription},
+    ]
+]
+
+inputs = processor.apply_chat_template(
+    conversation,
+    tokenize=True,
+    return_dict=True,
+    processor_kwargs={"output_labels": True},
+).to(model.device)
+
+outputs = model(**inputs)
+outputs.loss.backward()
+```
+</details>
+
+For more details about usage, please refer to the [Transformers' documentation](https://huggingface.co/docs/transformers/en/model_doc/canary).
+
+
 ## <span style="color:#b37800;">Software Integration</span>
 
 **Runtime Engine(s):**

chat_template.jinja

@@ -0,0 +1 @@
+{{- '<|startofcontext|><|startoftranscript|><|emo:undefined|>' -}}{%- for message in messages -%}{%- if message['role'] == 'user' -%}{%- for content in message['content'] if content['type'] == 'text' -%}{{- '<|' ~ content['source_language'] ~ '|>' -}}{{- '<|' ~ content['target_language'] ~ '|>' -}}{{- '<|pnc|>' if content['punctuation'] else '<|nopnc|>' -}}{{- '<|noitn|><|notimestamp|><|nodiarize|>' -}}{%- endfor -%}{%- elif message['role'] == 'assistant' -%}{{- message['content'] ~ '<|endoftext|>' -}}{%- endif -%}{%- endfor -%}

config.json

@@ -0,0 +1,61 @@
+{
+  "architectures": [
+    "CanaryForConditionalGeneration"
+  ],
+  "bos_token_id": 4,
+  "decoder_config": {
+    "attention_bias": true,
+    "attention_dropout": 0.1,
+    "bos_token_id": 4,
+    "eos_token_id": 3,
+    "head_dim": 128,
+    "hidden_act": "relu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "max_position_embeddings": 1024,
+    "model_type": "canary_decoder",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 8,
+    "num_key_value_heads": 8,
+    "pad_token_id": 2,
+    "use_cache": true,
+    "vocab_size": 16384
+  },
+  "decoder_start_token_id": 7,
+  "dtype": "float32",
+  "encoder_config": {
+    "activation_dropout": 0.1,
+    "attention_bias": true,
+    "attention_dropout": 0.1,
+    "conv_kernel_size": 9,
+    "convolution_bias": true,
+    "dropout": 0.1,
+    "dropout_positions": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layerdrop": 0.1,
+    "max_position_embeddings": 5000,
+    "model_type": "parakeet_encoder",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "num_mel_bins": 128,
+    "scale_input": false,
+    "subsampling_conv_channels": 256,
+    "subsampling_conv_kernel_size": 3,
+    "subsampling_conv_stride": 2,
+    "subsampling_factor": 8
+  },
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "is_encoder_decoder": true,
+  "model_type": "canary",
+  "pad_token_id": 2,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": true,
+  "vocab_size": 16384
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 4,
+  "decoder_start_token_id": 7,
+  "eos_token_id": 3,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 2,
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f6d381f6a939b95e9f8516212148b701e9b3805d26462c4c3e9e34f3d89b6f8
+size 3916173816

processor_config.json

@@ -0,0 +1,15 @@
+{
+  "feature_extractor": {
+    "feature_extractor_type": "ParakeetFeatureExtractor",
+    "feature_size": 128,
+    "hop_length": 160,
+    "n_fft": 512,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "preemphasis": 0.97,
+    "return_attention_mask": true,
+    "sampling_rate": 16000,
+    "win_length": 400
+  },
+  "processor_class": "CanaryProcessor"
+}

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftranscript|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "CanaryProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}