changes according to community

app.py

@@ -25,10 +25,13 @@ LICENSE = """
 if not torch.cuda.is_available():
     DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
 
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16, llm_int8_enable_fp32_cpu_offload=True
+)
 
 if torch.cuda.is_available():
     model_id = "mistral-community/Mixtral-8x22B-v0.1-4bit"
-    model = AutoModelForCausalLM.from_pretrained(model_id,quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=True),
+    model = AutoModelForCausalLM.from_pretrained(model_id,quantization_config = quantization_config,
                                                  device_map="cuda",
                                                  # torch_dtype=torch.float16,
                                                  # load_in_8bit=True,