test

app.py

@@ -3,7 +3,8 @@ import gradio as gr
 # from peft import AutoPeftModelForCausalLM
 # from transformers import AutoTokenizer, TextStreamer, BitsAndBytesConfig
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer
-
+from unsloth.chat_templates import get_chat_template
+from unsloth import FastLanguageModel
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
@@ -22,6 +23,12 @@ filename = "unsloth.Q4_K_M.gguf"
 tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
 model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)
 
+tokenizer = get_chat_template(
+    tokenizer,
+    chat_template = "llama-3.1",
+)
+FastLanguageModel.for_inference(model)
+
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -88,5 +95,5 @@ demo = gr.ChatInterface(
 )
 
 
-if __name__ == "__main__":
-    demo.launch()
+# if __name__ == "__main__":
+#     demo.launch()

requirements.txt

@@ -6,4 +6,5 @@ torch==2.5.1
 gguf==0.10.0
 sentencepiece==0.2.0
 numpy<2.0.0
-accelerate==1.1.1
+accelerate==1.1.1
+unsloth==2024.11.9