Upload README.md

README.md

@@ -118,6 +118,19 @@ model = AutoModelForCausalLM.from_pretrained(
     torch_dtype=torch_dtype,
     attn_implementation="sdpa",
 ).to(device)
+model.eval()
+
+
+def build_bad_words_ids(tokenizer):
+    eos_ids = tokenizer.eos_token_id
+    keep_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids or [])
+    bad_ids = set(tokenizer.all_special_ids) - keep_ids
+    bad_ids.update(
+        token_id
+        for token, token_id in tokenizer.get_added_vocab().items()
+        if token.startswith("<") and token.endswith(">") and token_id not in keep_ids
+    )
+    return [[token_id] for token_id in sorted(bad_ids)]
 
 conversation = [
     {
@@ -133,20 +146,25 @@ inputs = processor.apply_chat_template(
     conversation,
     add_generation_prompt=True,
     return_tensors="pt",
+    sampling_rate=16000,
+    audio_padding="longest",
+    text_kwargs={"padding": "longest"},
+    audio_max_length=30 * 16000,
 )
 inputs = inputs.to(device)
 if "audios" in inputs:
     inputs["audios"] = inputs["audios"].to(dtype=torch_dtype)
 
-bad_words_ids = [[token_id] for token_id in tokenizer.all_special_ids if token_id != tokenizer.eos_token_id]
-outputs = model.generate(
-    **inputs,
-    do_sample=False,
-    max_new_tokens=256,
-    pad_token_id=tokenizer.pad_token_id,
-    eos_token_id=tokenizer.eos_token_id,
-    bad_words_ids=bad_words_ids,
-)
+bad_words_ids = build_bad_words_ids(tokenizer)
+with torch.inference_mode():
+    outputs = model.generate(
+        **inputs,
+        do_sample=False,
+        max_new_tokens=256,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        bad_words_ids=bad_words_ids,
+    )
 decoded_outputs = tokenizer.batch_decode(
     outputs[:, inputs.input_ids.shape[1] :],
     skip_special_tokens=True,