Update model.py

model.py

@@ -18,7 +18,14 @@ if torch.cuda.is_available():
         use_safetensors=False,
     )
 else:
-    model = None
+    model = AutoModelForCausalLM.from_pretrained(
+        model_id,
+        config=config,
+        torch_dtype=torch.float16,
+        load_in_4bit=True,
+        device_map='auto',
+        use_safetensors=False,
+    )
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 
@@ -66,8 +73,8 @@ def run(message: str,
         temperature=temperature,
         num_beams=1,
     )
-    # t = Thread(target=model.generate, kwargs=generate_kwargs)
-    # t.start()
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
 
     outputs = []
     for text in streamer: