Update app.py

Added bits & bites config

app.py

@@ -2,10 +2,16 @@ import torch
 import gradio as gr
 
 from peft import PeftModel, PeftConfig
-from transformers import AutoModelForCausalLM, AutoTokenizer ,pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer ,pipeline, BitsAndBytesConfig
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16,
+)
 
 config = PeftConfig.from_pretrained("ShishuTripathi/entity_coder")
-model = AutoModelForCausalLM.from_pretrained("ybelkada/falcon-7b-sharded-bf16", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("ybelkada/falcon-7b-sharded-bf16", quantization_config=bnb_config,trust_remote_code=True)
 model = PeftModel.from_pretrained(model, "ShishuTripathi/entity_coder")
 tokenizer = AutoTokenizer.from_pretrained("ShishuTripathi/entity_coder")
 generator = pipeline('text-generation' , model = model, tokenizer =tokenizer, max_length = 50)