Trying 8bit from huggingface

app.py

@@ -3,16 +3,16 @@ import time
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import torch
 
-model_dir = "tinyllama_model"
-model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=torch.qint8)
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
+#model_dir = "tinyllama_model"
+model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 
+model = AutoModelForCausalLM.from_pretrained(model_id, load_in_8bit=True)
+#tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
-# Load the TinyLlama text generation pipeline
-pipe = pipeline("text-generation", model=model, torch_dtype=torch.qint8)
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
-pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, torch_dtype=torch.qint8)
+# Load the TinyLlama text generation pipeline
+pipe = pipeline("text-generation", model=model)
+#tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
 # Define the inference function
 def generate_text(prompt):