debug

data/models/llama3-1-70b.py

@@ -16,7 +16,7 @@ model_id = "meta-llama/Meta-Llama-3.1-70B-Instruct"
 pipeline = transformers.pipeline(
     "text-generation",
     model=model_id,
-    model_kwargs={"torch_dtype": torch.bfloat16},
+    model_kwargs={"torch_dtype": torch.bfloat16,"quantization_config": {"load_in_4bit": True}},
     device="cuda",
     token=True
 )