Update app.py

app.py

@@ -3,7 +3,7 @@ import gradio as gr
 from nltk.tokenize import sent_tokenize
 import torch
 import ujson as json
-from transformers import AutoModelForCausalLM,LlamaTokenizer
+from transformers import AutoModelForCausalLM,LlamaTokenizer,BitsAndBytesConfig
 from peft import PeftModel
 from keybert import KeyBERT
 from keyphrase_vectorizers import KeyphraseCountVectorizer
@@ -16,7 +16,14 @@ adapters_name = 'timdettmers/guanaco-7b'
 # print(f"Starting to load the model {model_name} into memory")
 m = AutoModelForCausalLM.from_pretrained(
     model_name,
-    torch_dtype=torch.bfloat16)
+    torch_dtype=torch.bfloat16,
+    quantization_config=BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_quant_type='nf4'
+    ),
+)
 m = PeftModel.from_pretrained(m, adapters_name)
 m = m.merge_and_unload()
 tok = LlamaTokenizer.from_pretrained(model_name)