test tokenizer

app.py

@@ -2,10 +2,7 @@ import gradio as gr
 # from huggingface_hub import InferenceClient
 # from peft import AutoPeftModelForCausalLM
 # from transformers import AutoTokenizer, TextStreamer, BitsAndBytesConfig
-# from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer
-import os
-os.environ["CUDA_VISIBLE_DEVICES"] = ""
-from unsloth import FastLanguageModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
@@ -18,23 +15,11 @@ For more information on `huggingface_hub` Inference API support, please check th
 # )
 # tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-# model_id = "EITD/model"
-# filename = "unsloth.Q4_K_M.gguf"
+model_id = "EITD/model"
+filename = "unsloth.Q4_K_M.gguf"
 
-# tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
-# model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)
-
-max_seq_length = 2048
-dtype = None # or torch.float32
-load_in_4bit = False
-
-model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name = "EITD/lora_model", # YOUR MODEL YOU USED FOR TRAINING
-        max_seq_length = max_seq_length,
-        dtype = dtype,
-        load_in_4bit = load_in_4bit,
-    )
-FastLanguageModel.for_inference(model)
+tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
+model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)
 
 def respond(
     message,
@@ -67,12 +52,14 @@ def respond(
 
     #     response += token
     #     yield response
-    inputs = tokenizer.apply_chat_template(
-        messages,
-        tokenize = True,
-        add_generation_prompt = True, # Must add for generation
-        return_tensors = "pt",
-    )
+    
+    # inputs = tokenizer.apply_chat_template(
+    #     messages,
+    #     tokenize = True,
+    #     add_generation_prompt = True, # Must add for generation
+    #     return_tensors = "pt",
+    # )
+    inputs = tokenizer.encode(messages, return_tensors='pt')
     # text_streamer = TextStreamer(tokenizer, skip_prompt = True)
     # for response in model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = max_tokens, use_cache = True,
     #                         temperature = temperature, min_p = top_p):