Update app.py

app.py

@@ -64,20 +64,28 @@ from huggingface_hub import login, HfFolder
 # model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_9",config=config, ignore_mismatched_sizes=True).to('cuda')
 # Load model directly
 
-tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini89", trust_remote_code=True)
-quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_use_double_quant=True,
-        bnb_4bit_quant_type="nf4",  
-        bnb_4bit_compute_dtype=torch.float16)
-model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini89", 
-                                             device_map="auto",
-                                             quantization_config=quantization_config, 
-                                             torch_dtype =torch.float16, 
-                                             low_cpu_mem_usage=True, 
-                                             use_safetensors=True,
-                                             trust_remote_code=True)
-
+# tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini89", trust_remote_code=True)
+# quantization_config = BitsAndBytesConfig(
+#         load_in_4bit=True,
+#         bnb_4bit_use_double_quant=True,
+#         bnb_4bit_quant_type="nf4",  
+#         bnb_4bit_compute_dtype=torch.float16)
+# model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini89", 
+#                                              device_map="auto",
+#                                              quantization_config=quantization_config, 
+#                                              torch_dtype =torch.float16, 
+#                                              low_cpu_mem_usage=True, 
+#                                              use_safetensors=True,
+#                                              trust_remote_code=True)
+from unsloth import FastLanguageModel
+model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name = "FlawedLLM/Bhashini_gemma_lora_clean_final", # YOUR MODEL YOU USED FOR TRAINING
+        max_seq_length = max_seq_length,
+        dtype = dtype,
+        load_in_4bit = load_in_4bit,)
+FastLanguageModel.for_inference(model) # Enable native 2x faster inference
+
+# alpaca_prompt = You MUST copy from above!
 @spaces.GPU(duration=300)
 def chunk_it(input_command, item_list):
     alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.