Update chat interface logic

app.py

@@ -137,21 +137,19 @@ MODEL_ID           = "JLee0/staria-pdf-chatbot-lora"
 BASE_ID = "MLP-KTLim/llama-3-Korean-Bllossom-8B"
 
 from peft import PeftModel
-
-tokenizer = AutoTokenizer.from_pretrained(BASE_ID)
-tokenizer.pad_token = tokenizer.eos_token
+import torch
 
 base_model = AutoModelForCausalLM.from_pretrained(
-    BASE_ID,
-    load_in_8bit=True,
-    device_map="auto"
-)
-model = PeftModel.from_pretrained(
-    base_model,
-    MODEL_ID,
-    device_map="auto"
-)
-
+        BASE_ID,
+        torch_dtype=torch.float16,
+        load_in_4bit=True,
+        device_map="auto"
+)    
+    
+model = PeftModel.from_pretrained(base_model, MODEL_ID)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "right"
 
 rag_embedder = SentenceTransformer("JLee0/rag-embedder-staria-10epochs")
 SYSTEM_PROMPT = PROMPTS["staria_after"]