Update app.py

app.py

@@ -2,13 +2,31 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 import transformers
 import torch
 
+from transformers import BitsAndBytesConfig
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+)
+
+model_id = "vilsonrodrigues/falcon-7b-instruct-sharded"
+
 # model = "tiiuae/falcon-40b-instruct"
-model = "tiiuae/falcon-7b-instruct"
+# model = "tiiuae/falcon-7b-instruct"
+
+model_4bit = AutoModelForCausalLM.from_pretrained(
+        model_id, 
+        device_map="auto",
+        quantization_config=quantization_config,
+        )
+tokenizer = AutoTokenizer.from_pretrained(model_id, )
 
-tokenizer = AutoTokenizer.from_pretrained(model, )
+# tokenizer = AutoTokenizer.from_pretrained(model, )
 pipeline = transformers.pipeline(
     "text-generation",
-    model=model,
+    # model=model,
+    model = model_4bit
     tokenizer=tokenizer,
     use_safetensors=True, 
     # torch_dtype=torch.bfloat16,