Support gated model: pass HF token to from_pretrained

handler.py

@@ -53,11 +53,18 @@ class EndpointHandler:
         os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
 
         # Load local repo (or remote if MODEL_ID points to hub id)
+        # Pass token when accessing gated repos
+        hub_token = os.environ.get("HUGGINGFACE_HUB_TOKEN") or os.environ.get("HF_HUB_TOKEN") or os.environ.get("HF_TOKEN")
+        load_kwargs = {
+            "trust_remote_code": True,
+            "torch_dtype": torch.bfloat16,
+            "device_map": "auto",
+        }
+        if hub_token:
+            load_kwargs["token"] = hub_token
         self.model = AutoModelForCausalLM.from_pretrained(
             model_id,
-            trust_remote_code=True,
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
+            **load_kwargs,
         )
 
         # Optional compilation for speed if exposed by remote code