test

app.py

@@ -3,7 +3,6 @@ import gradio as gr
 # from peft import AutoPeftModelForCausalLM
 # from transformers import AutoTokenizer, TextStreamer, BitsAndBytesConfig
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer
-from unsloth.chat_templates import get_chat_template
 from unsloth import FastLanguageModel
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
@@ -17,16 +16,22 @@ For more information on `huggingface_hub` Inference API support, please check th
 # )
 # tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-model_id = "EITD/model"
-filename = "unsloth.Q4_K_M.gguf"
+# model_id = "EITD/model"
+# filename = "unsloth.Q4_K_M.gguf"
 
-tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
-model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)
+# tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
+# model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)
 
-tokenizer = get_chat_template(
-    tokenizer,
-    chat_template = "llama-3.1",
-)
+max_seq_length = 2048
+dtype = None # or torch.float32
+load_in_4bit = False
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name = "EITD/lora_model", # YOUR MODEL YOU USED FOR TRAINING
+        max_seq_length = max_seq_length,
+        dtype = dtype,
+        load_in_4bit = load_in_4bit,
+    )
 FastLanguageModel.for_inference(model)
 
 def respond(
@@ -95,5 +100,5 @@ demo = gr.ChatInterface(
 )
 
 
-# if __name__ == "__main__":
-#     demo.launch()
+if __name__ == "__main__":
+    demo.launch()