Make the code robust against HTTP errors

src/details.py

@@ -55,6 +55,8 @@ async def load_details_dataframe(model_id, subtask):
         return
     path = max(paths)
     data = await load_jsonlines_file(path)
+    if not data:
+        return
     df = pd.json_normalize(data)
     # Keep model_name:
     df["model_name"] = model_id
@@ -70,6 +72,7 @@ async def load_details(subtask, *model_ids_lists):
             for model_id in model_ids
         ]
     )
+    dfs = [df for df in dfs if df is not None]
     if dfs:
         return pd.concat(dfs)
 

src/hub.py

@@ -8,7 +8,20 @@ from huggingface_hub.utils import build_hf_headers
 import src.constants as constants
 
 
-client = httpx.AsyncClient(follow_redirects=True)
+class Client:
+    def __init__(self):
+        self.client = httpx.AsyncClient(follow_redirects=True)
+
+    async def get(self, url, headers=None, params=None):
+        try:
+            r = await self.client.get(url, headers=headers, params=params)
+            r.raise_for_status()
+        except httpx.HTTPError:
+            return
+        return r
+
+
+client = Client()
 fs = HfFileSystem()
 
 
@@ -20,12 +33,16 @@ def glob(path):
 async def load_json_file(path):
     url = to_url(path)
     r = await client.get(url)
+    if r is None:
+        return
     return r.json()
 
 
 async def load_jsonlines_file(path):
     url = to_url(path)
     r = await client.get(url, headers=build_hf_headers())
+    if r is None:
+        return
     f = io.StringIO(r.text)
     return [json.loads(line) for line in f]
 
@@ -39,6 +56,8 @@ def to_url(path):
 async def load_model_card(model_id):
     url = to_url(f"{model_id}/README.md")
     r = await client.get(url)
+    if r is None:
+        return
     return ModelCard(r.text, ignore_metadata_errors=True)
 
 
@@ -47,4 +66,6 @@ async def list_models(filtering=None):
     if filtering:
         params["filter"] = filtering
     r = await client.get(f"{constants.HF_API_URL}/models", params=params)
+    if r is None:
+        return
     return r.json()

src/model_tree.py

@@ -30,13 +30,17 @@ async def load_model_tree(result_paths_per_model, model_ids):
 
 async def load_base_models(model_id) -> list[str]:
     card = await load_model_card(model_id)
+    if not card:
+        return []
     base_models = getattr(card.data, constants.BASE_MODEL_TYPE[1])
     if not isinstance(base_models, list):
         base_models = [base_models]
     return base_models
 
 
-async def load_derived_models_by_type(model_id, derived_model_type):
+async def load_derived_models_by_type(model_id, derived_model_type) -> list[str]:
     models = await list_models(filtering=f"base_model:{derived_model_type}:{model_id}")
+    if not models:
+        return []
     models = [model["id"] for model in models]
     return models

src/results.py

@@ -34,6 +34,9 @@ async def load_results_dataframe(model_id, result_paths_per_model=None):
         return
     result_paths = result_paths_per_model[model_id]
     results = await asyncio.gather(*[load_json_file(path) for path in result_paths])
+    results = [result for result in results if result]
+    if not results:
+        return
     data = {"results": {}, "configs": {}}
     for result in results:
         data["results"].update(result["results"])