Update app.py

app.py

@@ -13,20 +13,27 @@ from peft import PeftModel
 import transformers
 
 from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig
+from transformers import BitsAndBytesConfig
 
 tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
 
+
+
+quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=True)
+
 model = LlamaForCausalLM.from_pretrained(
     "decapoda-research/llama-7b-hf",
-    load_in_8bit=True,
-    torch_dtype=torch.float16,
-    # device_map="auto",
-    device_map={"":"cpu"},
-    max_memory={"cpu":"12GiB"}
+    # load_in_8bit=True,
+    # torch_dtype=torch.float16,
+    device_map="auto",
+    # device_map={"":"cpu"},
+    max_memory={"cpu":"15GiB"}
+    quantization_config=quantization_config
 )
 model = PeftModel.from_pretrained(
     model, "tloen/alpaca-lora-7b",
-    torch_dtype=torch.float16
+    # torch_dtype=torch.float16,
+    device_map={"":"cpu"},
 )
 
 device = "cpu"