back to community

app.py

@@ -27,10 +27,11 @@ if not torch.cuda.is_available():
 
 
 if torch.cuda.is_available():
-    model_id = "mistralai/Mixtral-8x7B-v0.1"
+    model_id = "mistral-community/Mixtral-8x22B-v0.1-4bit"
     model = AutoModelForCausalLM.from_pretrained(model_id,
+                                                 device_map="auto",
                                                  # torch_dtype=torch.float16,
-                                                 load_in_8bit=True,
+                                                 # load_in_8bit=True,
                                                  trust_remote_code=True).to("cuda")
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     tokenizer.use_default_system_prompt = False