Update for CPU: float(), requried

app.py

@@ -27,8 +27,8 @@ except Exception:
     # Windows
     logger.warning("Windows, cant run time.tzset()")
 
-# model_name = "THUDM/chatglm2-6b"
-model_name = "THUDM/chatglm2-6b-int4"
+# model_name = "THUDM/chatglm2-6b"  # 7x?G
+model_name = "THUDM/chatglm2-6b-int4"  # 3.9G
 
 RETRY_FLAG = False
 
@@ -43,13 +43,16 @@ has_cuda = torch.cuda.is_available()
 # has_cuda = False  # force cpu
 
 if has_cuda:
-    model = (
-        AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
-    )  # 3.92G
+    if model_name.endswith("int4"):
+        model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
+    else:
+        model = (
+            AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
+        )
 else:
     model = AutoModel.from_pretrained(
         model_name, trust_remote_code=True
-    ).half()  # .float() .half().float()
+    ).float()  #  .half().float(), .float() required for CPU
 
 model = model.eval()