Back to Qwen/Qwen-7B-Chat

app.py

@@ -60,8 +60,8 @@ except Exception:
     # Windows
     logger.warning("Windows, cant run time.tzset()")
 
-model_name = "Qwen/Qwen-7B-Chat"  # gone!
 model_name = "tangger/Qwen-7B-Chat"  # try
+model_name = "Qwen/Qwen-7B-Chat"  # gone!
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 
 n_gpus = torch.cuda.device_count()
@@ -82,7 +82,8 @@ def gen_model(model_name: str):
         model_name,
         trust_remote_code=True,
         device_map="auto",
-        load_in_4bit=True,
+        # load_in_4bit=True,
+        load_in_8bit=True,
         max_memory=max_memory,
         fp16=True,
         torch_dtype=torch.float16,