debug

data/models/llama3-1-70b.py

@@ -14,12 +14,15 @@ login(api_key,add_to_git_credential=True)
 
 model_id = "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4"
 
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForCausalLM.from_pretrained(
-  model_id,
-  torch_dtype=torch.float16,
-  low_cpu_mem_usage=True,
-).to("cuda")
+tokenizer   = AutoTokenizer.from_pretrained(model_id)
+model       = AutoModelForCausalLM.from_pretrained(
+                model_id,
+                torch_dtype=torch.float16,
+                low_cpu_mem_usage=True,
+                device_map="sequential",
+                offload_folder="offload",
+                offload_state_dict=True 
+            ).to("cuda")
 
 @app.route('/chat', methods=['POST'])
 @spaces.GPU()