Document verified concise four-sentence inference config

README.md

@@ -52,11 +52,35 @@ included in this repository.
 ## Usage
 
 ```python
+import re
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    StoppingCriteria,
+    StoppingCriteriaList,
+)
+
+
+class SentenceLimitCriteria(StoppingCriteria):
+    """Stop after a fixed number of complete generated sentences."""
+
+    def __init__(self, tokenizer, prompt_length, max_sentences=4):
+        self.tokenizer = tokenizer
+        self.prompt_length = prompt_length
+        self.max_sentences = max_sentences
+
+    def __call__(self, input_ids, scores, **kwargs):
+        generated = self.tokenizer.decode(
+            input_ids[0, self.prompt_length:], skip_special_tokens=True
+        )
+        endings = re.findall(r'[.!?](?:["\'’”)]*)?\s+', generated)
+        return len(endings) >= self.max_sentences
+
 
 model_id = "NeuronUz/qwen3.5-2b-fine-tuned"
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
+max_sentences = 4
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
@@ -67,7 +91,14 @@ model = AutoModelForCausalLM.from_pretrained(
 )
 
 messages = [
-    {"role": "system", "content": "Siz foydali AI yordamchisiz."},
+    {
+        "role": "system",
+        "content": (
+            "Siz foydali AI yordamchisiz. Javoblarni qisqa va aniq yozing. "
+            "Agar foydalanuvchi batafsil javob so'ramasa, odatda 2-4 ta "
+            "to'liq gap bilan javob bering."
+        ),
+    },
     {"role": "user", "content": "O'zbekiston haqida qisqacha ma'lumot bering."},
 ]
 
@@ -78,15 +109,43 @@ inputs = tokenizer.apply_chat_template(
     return_dict=True,
 ).to(model.device)
 
+im_end_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
+eos_ids = [tokenizer.eos_token_id, im_end_id]
+stopping_criteria = StoppingCriteriaList(
+    [
+        SentenceLimitCriteria(
+            tokenizer,
+            prompt_length=inputs["input_ids"].shape[-1],
+            max_sentences=max_sentences,
+        )
+    ]
+)
+
 with torch.inference_mode():
     output = model.generate(
         **inputs,
         max_new_tokens=256,
         do_sample=False,
+        repetition_penalty=1.15,
+        no_repeat_ngram_size=3,
+        eos_token_id=eos_ids,
+        pad_token_id=tokenizer.eos_token_id,
+        stopping_criteria=stopping_criteria,
     )
 
 prompt_length = inputs["input_ids"].shape[-1]
-print(tokenizer.decode(output[0][prompt_length:], skip_special_tokens=True))
+reply = tokenizer.decode(
+    output[0][prompt_length:], skip_special_tokens=True
+).strip()
+
+# A token can contain the final period and the start of the next word, so trim
+# the displayed output back to the fourth complete sentence.
+sentence_end_re = re.compile(r'[.!?](?:["\'’”)]*)?(?=\s|$)')
+sentence_endings = list(sentence_end_re.finditer(reply))
+if len(sentence_endings) >= max_sentences:
+    reply = reply[:sentence_endings[max_sentences - 1].end()].strip()
+
+print(reply)
 ```
 
 Use a recent Transformers release with Qwen3.5 support.
@@ -94,8 +153,10 @@ Use a recent Transformers release with Qwen3.5 support.
 When multiple GPUs are visible, avoid `device_map="auto"` with this checkpoint.
 Current Accelerate/Transformers releases may split the Qwen3.5 hybrid layers
 across GPUs and produce invalid text. Pin the complete model to one GPU as shown
-above. If sampling is desired, a tested starting point is `temperature=0.7`,
-`top_p=0.8`, and `top_k=20`.
+above. The example uses greedy decoding (`do_sample=False`, equivalent to
+temperature 0 in the local chat script) and limits normal answers to four
+complete sentences. If sampling is desired, a tested starting point is
+`temperature=0.7`, `top_p=0.8`, and `top_k=20`.
 
 ## Limitations