Load Details asynchronously

src/details.py

@@ -1,10 +1,11 @@
-import json
+import asyncio
 
 import gradio as gr
 import pandas as pd
 from huggingface_hub import HfFileSystem
 
 from src.constants import SUBTASKS, DETAILS_DATASET_ID, DETAILS_FILENAME
+from src.hub import load_details_file
 
 
 def update_subtasks_component(task):
@@ -22,7 +23,7 @@ def update_load_details_component(model_id_1, model_id_2, subtask):
         return gr.Button("Load Details", interactive=False)
 
 
-def load_details_dataframe(model_id, subtask):
+async def load_details_dataframe(model_id, subtask):
     fs = HfFileSystem()
     if not model_id or not subtask:
         return
@@ -35,8 +36,7 @@ def load_details_dataframe(model_id, subtask):
     if not paths:
         return
     path = max(paths)
-    with fs.open(path, "r") as f:
-        data = [json.loads(line) for line in f]
+    data = await load_details_file(path)
     df = pd.json_normalize(data)
     # df = df.rename_axis("Parameters", axis="columns")
     df["model_name"] = model_id  # Keep model_name
@@ -44,8 +44,9 @@ def load_details_dataframe(model_id, subtask):
     # return df.set_index(pd.Index([model_id])).reset_index()
 
 
-def load_details_dataframes(subtask, *model_ids):
-    return [load_details_dataframe(model_id, subtask) for model_id in model_ids]
+async def load_details_dataframes(subtask, *model_ids):
+    result = await asyncio.gather(*[load_details_dataframe(model_id, subtask) for model_id in model_ids])
+    return result
 
 
 def display_details(sample_idx, *dfs):

src/hub.py

@@ -1,6 +1,8 @@
 import httpx
 from huggingface_hub import hf_hub_url
+from huggingface_hub.utils import build_hf_headers
 
+import json
 
 client = httpx.AsyncClient()
 
@@ -11,6 +13,12 @@ async def load_file(path):
     return r.json()
 
 
+async def load_details_file(path):
+    url = to_url(path)
+    r = await client.get(url, headers=build_hf_headers())
+    return [json.loads(line) for line in r.text.splitlines()]
+
+
 def to_url(path):
     _, org_name, ds_name, filename = path.split("/", 3)
     return hf_hub_url(repo_id=f"{org_name}/{ds_name}", filename=filename, repo_type="dataset")