Update app.py

app.py

@@ -5,11 +5,9 @@ import re
 import numpy as np
 import json
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+from transformers_stream_generator import StreamingGenerationConfig
 import gradio as gr
 
-# Применяем патч для streaming
-patch_streaming()
-
 # === Настройки ===
 model_name = "Qwen/Qwen2.5-0.5B-Instruct"
 
@@ -85,21 +83,25 @@ def generate_stream(prompt):
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 
-    def generate():
-        with torch.no_grad():
-            model.generate(
-                **inputs,
-                max_new_tokens=256,
-                temperature=0.7,
-                streamer=streamer,
-                use_cache=True
-            )
+    # Используем специальную конфигурацию
+    generation_config = StreamingGenerationConfig(
+        max_new_tokens=256,
+        temperature=0.7,
+        top_p=0.9,
+        do_sample=True
+    )
 
-    thread = threading.Thread(target=generate)
+    thread = threading.Thread(target=model.generate, kwargs={
+        **inputs,
+        "streamer": streamer,
+        "generation_config": generation_config
+    })
     thread.start()
 
-    for text in streamer:
-        yield text
+    buffer = ""
+    for token in streamer:
+        buffer += token
+        yield buffer
 
 # === Gradio ===
 with gr.Blocks() as demo: