switch loading method

app.py

@@ -1,6 +1,8 @@
 from dash import Dash, html, dcc, Input, Output, State
 import pandas as pd
 import dash_mantine_components as dmc
+from dataasets import load_dataset
+import time
 from graphs.leaderboard import (
     create_leaderboard,
     get_top_n_leaderboard,
@@ -12,10 +14,26 @@ app = Dash()
 server = app.server
 
 # Load parquet file from Hugging Face
-print("Loading data...")
-hf_parquet_url = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/filtered_df.parquet"
-filtered_df = pd.read_parquet(hf_parquet_url)
-print("Data loaded.")
+HF_DATASET_ID = "emsesc/open_model_evolution_data"
+hf_parquet_url = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/"
+data_files = {
+    "filtered_df": hf_parquet_url + "filtered_df.parquet",
+    # "weekly_df": hf_parquet_url + "weekly_df.parquet",
+}
+filtered_df = pd.DataFrame()
+
+print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
+try:
+    overall_start_time = time.time()
+    dataset = load_dataset("parquet", data_files=data_files)
+    df = dataset["filtered_df"].to_pandas()
+    filtered_df = df.copy()
+
+    msg = f"Successfully loaded dataset in {time.time() - overall_start_time:.2f}s."
+    print(msg)
+except Exception as e:
+    err_msg = f"Failed to load dataset. Error: {e}"
+    print(err_msg)
 
 # List columns for reference
 print(filtered_df.columns.tolist())

requirements.txt

@@ -5,4 +5,5 @@ gunicorn
 dash-mantine-components
 dash-bootstrap-components
 pyarrow
-dash-iconify
+dash-iconify
+datasets