Update app.py

app.py

@@ -5,7 +5,7 @@ from typing import Iterator
 import gradio as gr
 import spaces
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, StoppingCriteria, StoppingCriteriaList
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, StoppingCriteria, StoppingCriteriaList, BitsAndBytesConfig
 
 # DESCRIPTION = """\
 # # FinID
@@ -35,6 +35,7 @@ Your responses should only be within the financial subject, any other prompt or
 # DEFAULT_MAX_NEW_TOKENS = 512
 
 model_id = "FOLZi/FinID_v2_8B_Chat"
+quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 
 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
@@ -42,6 +43,7 @@ model = AutoModelForCausalLM.from_pretrained(
     trust_remote_code=True,
     device_map="auto",
     torch_dtype=torch.float16,
+    quantization_config=quantization_config
 )
 # model.config.sliding_window = 4096
 # model.eval()