Back to a10g

app.py

@@ -20,9 +20,8 @@ print(f"Starting to load the model {model_name}")
 
 m = AutoModelForCausalLM.from_pretrained(
     model_name,
-    device_map='auto',
+    device_map=0,
     #load_in_8bit=True,
-    llm_int8_enable_fp32_cpu_offload=True
 )
 
 m.config.pad_token_id = m.config.eos_token_id