Fix inference example and multi-GPU device placement

README.md

@@ -56,12 +56,14 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 model_id = "NeuronUz/qwen3.5-2b-fine-tuned"
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     torch_dtype="auto",
-    device_map="auto",
+    # Keep this hybrid model on one device. See the note below.
+    device_map=device,
 )
 
 messages = [
@@ -73,26 +75,31 @@ inputs = tokenizer.apply_chat_template(
     messages,
     add_generation_prompt=True,
     return_tensors="pt",
+    return_dict=True,
 ).to(model.device)
 
 with torch.inference_mode():
     output = model.generate(
-        inputs,
+        **inputs,
         max_new_tokens=256,
-        do_sample=True,
-        temperature=0.7,
-        top_p=0.9,
+        do_sample=False,
     )
 
-print(tokenizer.decode(output[0][inputs.shape[-1]:], skip_special_tokens=True))
+prompt_length = inputs["input_ids"].shape[-1]
+print(tokenizer.decode(output[0][prompt_length:], skip_special_tokens=True))
 ```
 
 Use a recent Transformers release with Qwen3.5 support.
 
+When multiple GPUs are visible, avoid `device_map="auto"` with this checkpoint.
+Current Accelerate/Transformers releases may split the Qwen3.5 hybrid layers
+across GPUs and produce invalid text. Pin the complete model to one GPU as shown
+above. If sampling is desired, a tested starting point is `temperature=0.7`,
+`top_p=0.8`, and `top_k=20`.
+
 ## Limitations
 
 The model may produce inaccurate, biased, or fabricated information. It has not
 been comprehensively evaluated for safety or high-stakes domains. Outputs should
 be independently verified before use in medical, legal, financial, or other
 consequential settings.
-