Use HF api to check repo existance

utils.py

@@ -7,6 +7,10 @@ from urllib.parse import urljoin, urlparse
 import requests
 from bs4 import BeautifulSoup
 
+from huggingface_hub import HfApi
+
+HF_API = HfApi()
+
 
 def get_base_url(url: str) -> str:
     """
@@ -129,7 +133,7 @@ def build_dataset_url(dataset_name: str):
     """
     url = f"https://huggingface.co/datasets/{dataset_name}"
     # Test if the url exists
-    if url_exists(url):
+    if url_exists(url) and HF_API.repo_exists(dataset_name, repo_type="dataset"):
         return url
     else:
         return None
@@ -141,7 +145,7 @@ def build_model_url(model_name: str):
     """
     url = f"https://huggingface.co/{model_name}"
     # Test if the url exists
-    if url_exists(url):
+    if url_exists(url) and HF_API.repo_exists(model_name, repo_type="model"):
         return url
     else:
         return None