Update main.py

main.py

@@ -1,13 +1,13 @@
 from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline,BitsAndBytesConfig
-import accelerate
-import bitsandbytes
+# import accelerate
+# import bitsandbytes
 from langchain_core.prompts import PromptTemplate
 
-quants = BitsAndBytesConfig(load_in_4bit=True)
+# quants = BitsAndBytesConfig(load_in_4bit=True)
 model_id = "mistralai/Mistral-7B-Instruct-v0.2"
-tokenizer = AutoTokenizer.from_pretrained(model_id,quantization_config=quants)
-model = AutoModelForCausalLM.from_pretrained(model_id,quantization_config=quants)
+tokenizer = AutoTokenizer.from_pretrained(model_id,load_in_8bit=True)
+model = AutoModelForCausalLM.from_pretrained(model_id,load_in_8bit=True)
 pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
 hf = HuggingFacePipeline(pipeline=pipe)