Updates

app.py

@@ -41,11 +41,11 @@ class ChatBot:
         self.bot_input_ids = None
         self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
         self.tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0", use_fast=False)
-        self.model = AutoModelForCausalLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0", quantization_config=bnb_config, device_map="cpu")
-        self.model = quantize_dynamic(self.model, {torch.nn.Linear}, dtype=torch.qint8)
+        self.model = AutoModelForCausalLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
+        self.model = torch.ao.quantization.quantize_dynamic(self.model, {torch.nn.Linear}, dtype=torch.qint8)
         # Parler-TTS Setup
-        self.tts_model = ParlerTTSForConditionalGeneration.from_pretrained("doublesizebed/parler-tts-mini-malay").to("cpu")
-        self.tts_model = quantize_dynamic(self.tts_model, {torch.nn.Linear}, dtype=torch.qint8)
+        self.tts_model = ParlerTTSForConditionalGeneration.from_pretrained("doublesizebed/parler-tts-mini-malay")
+        self.tts_model = torch.ao.quantization.quantize_dynamic(self.tts_model, {torch.nn.Linear}, dtype=torch.qint8)
         self.tts_tokenizer = AutoTokenizer.from_pretrained("doublesizebed/parler-tts-mini-malay")
         self.description_tokenizer = AutoTokenizer.from_pretrained(self.tts_model.config.text_encoder._name_or_path)
     

requirements.txt

@@ -12,6 +12,6 @@ textblob
 malaya
 parler_tts
 nltk
-bitsandbytes
+bitsandbytes==0.45.5
 accelerate
 accelerate[torch]