Update app.py

app.py

@@ -1,12 +1,11 @@
 # app.py
-# =======
-# Полная версия исправленного кода приложения для генерации текста с использованием Gradio 4.44.1
-# и модели Qwen/Qwen2.5-Coder-0.5B-Instruct.
+# =============
+# This is a complete app.py file for a text generation app using the Qwen/Qwen2.5-Coder-0.5B-Instruct model.
+# The app uses the Gradio library to create a web interface for interacting with the model.
 
 # Imports
 # =======
 import gradio as gr
-import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 # Constants
@@ -20,12 +19,11 @@ def load_model_and_tokenizer():
     """
     Load the model and tokenizer from Hugging Face.
     """
-    device = "cuda" if torch.cuda.is_available() else "cpu"
     tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
     model = AutoModelForCausalLM.from_pretrained(
         MODEL_NAME,
-        torch_dtype=torch.float16 if device == "cuda" else torch.float32,
-        device_map=device
+        torch_dtype="auto",
+        device_map="cpu"  # Ensure the model runs on the CPU
     )
     return model, tokenizer
 
@@ -39,9 +37,12 @@ def generate_response(prompt, chat_history, max_new_tokens, temperature):
     Generate a response from the model based on the user prompt and chat history.
     """
     messages = [{"role": "system", "content": SYSTEM_MESSAGE}] + chat_history + [{"role": "user", "content": prompt}]
-    # Concatenate messages into a single string for the model
-    text = "\n".join(f"{msg['role']}: {msg['content']}" for msg in messages)
-    model_inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=1024).to(model.device)
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
 
     generated_ids = model.generate(
         **model_inputs,
@@ -49,11 +50,16 @@ def generate_response(prompt, chat_history, max_new_tokens, temperature):
         do_sample=True,
         top_k=50,
         top_p=0.95,
-        temperature=temperature
+        temperature=temperature,
+        output_scores=True,
+        return_dict_in_generate=True,
+        return_legacy_cache=True  # Ensure legacy format is returned
     )
 
-    response = tokenizer.decode(generated_ids[0][model_inputs.input_ids.shape[1]:], skip_special_tokens=True)
-    return response
+    response = ""
+    for token_id in generated_ids.sequences[0][len(model_inputs.input_ids[0]):]:
+        response += tokenizer.decode([token_id], skip_special_tokens=True)
+        yield chat_history + [{"role": "assistant", "content": response}]
 
 # Clear Chat History
 # ==================
@@ -84,12 +90,13 @@ def gradio_interface():
                     temperature = gr.Slider(0.1, 1.0, value=0.7, step=0.05, label="Temperature")
 
         def respond(message, chat_history, max_new_tokens, temperature):
-            if not message.strip():
-                return chat_history, ""
             chat_history.append({"role": "user", "content": message})
-            response = generate_response(message, chat_history, max_new_tokens, temperature)
+            response = ""
+            for chunk in generate_response(message, chat_history, max_new_tokens, temperature):
+                response = chunk[-1]["content"]
+                yield chat_history, ""
             chat_history.append({"role": "assistant", "content": response})
-            return chat_history, ""
+            yield chat_history, ""
 
         submit.click(respond, [msg, chatbot, max_new_tokens, temperature], [chatbot, msg])
         msg.submit(respond, [msg, chatbot, max_new_tokens, temperature], [chatbot, msg])
@@ -104,4 +111,11 @@ if __name__ == "__main__":
 
 # Dependencies
 # =============
-# pip install transformers gradio==4.44.1 torch accelerate
+# The following dependencies are required to run this app:
+# - transformers
+# - gradio
+# - torch
+# - accelerate
+#
+# You can install these dependencies using pip:
+# pip install transformers gradio torch accelerate