Update app to pull local quantized model

app.py

@@ -1,10 +1,18 @@
 import gradio as gr
 import time
-from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import torch
 
+model_dir = "tinyllama_model"
+model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype=torch.qint8)
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+
+
 # Load the TinyLlama text generation pipeline
-pipe = pipeline("text-generation", model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", torch_dtype=torch.int8)
+pipe = pipeline("text-generation", model=model, torch_dtype=torch.qint8)
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+
+pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, torch_dtype=torch.qint8)
 
 # Define the inference function
 def generate_text(prompt):