5

app.py

@@ -1,10 +1,10 @@
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
 
-# Загружаем Mistral-7B
-model_name = "mistralai/Mistral-7B-Instruct-v0.3"
+model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
 
 def chat(message):
     inputs = tokenizer(message, return_tensors="pt").to("cpu")
@@ -12,6 +12,5 @@ def chat(message):
     response = tokenizer.decode(outputs[0], skip_special_tokens=True)
     return response
 
-iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="Mistral-7B Chatbot")
+iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="TinyLlama Chatbot")
 iface.launch()
-

requirements.txt

@@ -1,6 +1,5 @@
+torch
 transformers
 gradio
-torch
-bitsandbytes
+accelerate
 sentencepiece
-accelerate>=0.26.0