init pass

app.py

@@ -1,6 +1,8 @@
 import os
 import json
 import requests
+import glob
+from pathlib import Path
 
 import gradio as gr
 import pandas as pd
@@ -12,290 +14,106 @@ from tqdm.contrib.concurrent import thread_map
 
 from utils import *
 
-DATASET_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/drlc-leaderboard-data"
-DATASET_REPO_ID = "huggingface-projects/drlc-leaderboard-data"
+# Configuration for retrieval task leaderboard
+SUBMISSION_FOLDER = "submission"
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
 block = gr.Blocks()
 api = HfApi(token=HF_TOKEN)
 
-# Containing the data
-rl_envs = [
-{
-"rl_env_beautiful": "LunarLander-v2 🚀",
-"rl_env": "LunarLander-v2",
-"video_link": "",
-"global": None
-},    
-{
-"rl_env_beautiful": "CartPole-v1",
-"rl_env": "CartPole-v1",
-"video_link": "https://huggingface.co/sb3/ppo-CartPole-v1/resolve/main/replay.mp4",
-"global": None
-},
-{
-"rl_env_beautiful": "FrozenLake-v1-4x4-no_slippery ❄️",
-"rl_env": "FrozenLake-v1-4x4-no_slippery",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "FrozenLake-v1-8x8-no_slippery ❄️",
-"rl_env": "FrozenLake-v1-8x8-no_slippery",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "FrozenLake-v1-4x4 ❄️",
-"rl_env": "FrozenLake-v1-4x4",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "FrozenLake-v1-8x8 ❄️",
-"rl_env": "FrozenLake-v1-8x8",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "Taxi-v3 🚖",
-"rl_env": "Taxi-v3",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "CarRacing-v0 🏎️",
-"rl_env": "CarRacing-v0",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "CarRacing-v2 🏎️",
-"rl_env": "CarRacing-v2",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "MountainCar-v0 ⛰️",
-"rl_env": "MountainCar-v0",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "SpaceInvadersNoFrameskip-v4 👾",
-"rl_env": "SpaceInvadersNoFrameskip-v4",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "PongNoFrameskip-v4 🎾",
-"rl_env": "PongNoFrameskip-v4",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "BreakoutNoFrameskip-v4 🧱",
-"rl_env": "BreakoutNoFrameskip-v4",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "QbertNoFrameskip-v4 🐦",
-"rl_env": "QbertNoFrameskip-v4",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "BipedalWalker-v3",
-"rl_env": "BipedalWalker-v3",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "Walker2DBulletEnv-v0",
-"rl_env": "Walker2DBulletEnv-v0",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "AntBulletEnv-v0",
-"rl_env": "AntBulletEnv-v0",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "HalfCheetahBulletEnv-v0",
-"rl_env": "HalfCheetahBulletEnv-v0",
-"video_link": "",
-"global": None
-},
-{
-"rl_env_beautiful": "PandaReachDense-v2",
-"rl_env": "PandaReachDense-v2",
-"video_link": "",
-"global": None
-},  
-{
-"rl_env_beautiful": "PandaReachDense-v3",
-"rl_env": "PandaReachDense-v3",
-"video_link": "",
-"global": None
-},  
-{
-"rl_env_beautiful": "Pixelcopter-PLE-v0",
-"rl_env": "Pixelcopter-PLE-v0",
-"video_link": "",
-"global": None
-}
+# Retrieval task metrics configuration
+retrieval_metrics = [
+    {
+        "metric_name": "Hit Rate Click@50",
+        "metric_key": "hit_rate_click@50",
+        "description": "Hit rate for click predictions at top 50"
+    },
+    {
+        "metric_name": "Hit Rate A2C@50", 
+        "metric_key": "hit_rate_A2C@50",
+        "description": "Hit rate for A2C predictions at top 50"
+    },
+    {
+        "metric_name": "Hit Rate Purchase@50",
+        "metric_key": "hit_rate_purchase@50", 
+        "description": "Hit rate for purchase predictions at top 50"
+    }
 ]
 
+# Main leaderboard configuration
+leaderboard_config = {
+    "title": "🏆 Retrieval Task Leaderboard 🏆",
+    "description": "Leaderboard for retrieval task performance",
+    "metrics": retrieval_metrics
+}
+
 def restart():
     print("RESTART")
     api.restart_space(repo_id="huggingface-projects/Deep-Reinforcement-Learning-Leaderboard")
 
-def get_metadata(model_id):
+def load_submission_files():
+    """Load all JSON submission files from the submission folder"""
+    submission_files = glob.glob(os.path.join(SUBMISSION_FOLDER, "*.json"))
+    submissions = []
+    
+    for file_path in submission_files:
+        try:
+            with open(file_path, 'r') as f:
+                submission_data = json.load(f)
+                # Validate required fields
+                required_fields = ["user_id", "model_id", "hit_rate_click@50", "hit_rate_A2C@50", "hit_rate_purchase@50"]
+                if all(field in submission_data for field in required_fields):
+                    submissions.append(submission_data)
+                else:
+                    print(f"Warning: Invalid submission format in {file_path}")
+        except (json.JSONDecodeError, FileNotFoundError) as e:
+            print(f"Error reading {file_path}: {e}")
+    
+    return submissions
+
+def parse_submission_data(submission):
+    """Parse a single submission and return formatted data"""
     try:
-        readme_path = hf_hub_download(model_id, filename="README.md", etag_timeout=180)
-        return metadata_load(readme_path)
-    except requests.exceptions.HTTPError:
-        # 404 README.md not found
-        return None
+        # Convert string metrics to float, handle potential errors
+        click_rate = float(submission.get("hit_rate_click@50", 0))
+        a2c_rate = float(submission.get("hit_rate_A2C@50", 0))
+        purchase_rate = float(submission.get("hit_rate_purchase@50", 0))
         
-def parse_metrics_accuracy(meta):
-    if "model-index" not in meta:
+        return {
+            "User": submission.get("user_id", "Unknown"),
+            "Model": submission.get("model_id", "Unknown"),
+            "Dataset": submission.get("dataset_id", "Unknown"),
+            "Hit Rate Click@50": click_rate,
+            "Hit Rate A2C@50": a2c_rate,
+            "Hit Rate Purchase@50": purchase_rate,
+            "Comment": submission.get("comment", "")
+        }
+    except (ValueError, TypeError) as e:
+        print(f"Error parsing submission data: {e}")
         return None
-    result = meta["model-index"][0]["results"]
-    metrics = result[0]["metrics"]
-    accuracy = metrics[0]["value"]
-    return accuracy
-
-# We keep the worst case episode
-def parse_rewards(accuracy):
-    default_std = -1000
-    default_reward=-1000
-    if accuracy !=  None:
-        accuracy = str(accuracy)
-        parsed =  accuracy.split('+/-')
-        if len(parsed)>1:
-            mean_reward = float(parsed[0].strip())
-            std_reward =  float(parsed[1].strip())
-        elif len(parsed)==1: #only mean reward
-            mean_reward = float(parsed[0].strip())
-            std_reward =  float(0) 
-        else: 
-            mean_reward = float(default_std)
-            std_reward = float(default_reward)
-
-    else:
-        mean_reward = float(default_std)
-        std_reward = float(default_reward)
-    return mean_reward, std_reward
-
 
-def get_model_ids(rl_env):
-    api = HfApi()
-    models = api.list_models(filter=rl_env)
-    model_ids = [x.modelId for x in models]
-    return model_ids
-
-# Parralelized version
-def update_leaderboard_dataset_parallel(rl_env, path):
-    # Get model ids associated with rl_env
-    model_ids = get_model_ids(rl_env)
-
-    def process_model(model_id):
-        meta = get_metadata(model_id)
-        #LOADED_MODEL_METADATA[model_id] = meta if meta is not None else ''
-        if meta is None:
-            return None
-        user_id = model_id.split('/')[0]
-        row = {}
-        row["User"] = user_id
-        row["Model"] = model_id
-        accuracy = parse_metrics_accuracy(meta)
-        mean_reward, std_reward = parse_rewards(accuracy)
-        mean_reward = mean_reward if not pd.isna(mean_reward) else 0
-        std_reward = std_reward if not pd.isna(std_reward) else 0
-        row["Results"] = mean_reward - std_reward
-        row["Mean Reward"] = mean_reward
-        row["Std Reward"] = std_reward
-        return row
-
-    data = list(thread_map(process_model, model_ids, desc="Processing models"))
-
-    # Filter out None results (models with no metadata)
-    data = [row for row in data if row is not None]
-
-    ranked_dataframe = rank_dataframe(pd.DataFrame.from_records(data))
-    new_history = ranked_dataframe
-    file_path = path + "/" + rl_env + ".csv"
-    new_history.to_csv(file_path, index=False)
-
-    return ranked_dataframe
-
-
-def update_leaderboard_dataset(rl_env, path):
-    # Get model ids associated with rl_env
-    model_ids = get_model_ids(rl_env)
+def update_leaderboard_from_submissions():
+    """Update leaderboard data from JSON submissions"""
+    submissions = load_submission_files()
     data = []
-    for model_id in model_ids:
-        """
-        readme_path = hf_hub_download(model_id, filename="README.md")
-        meta = metadata_load(readme_path)
-        """
-        meta = get_metadata(model_id)
-        #LOADED_MODEL_METADATA[model_id] = meta if meta is not None else ''
-        if meta is None:
-            continue
-        user_id = model_id.split('/')[0]
-        row = {}
-        row["User"] = user_id
-        row["Model"] = model_id
-        accuracy = parse_metrics_accuracy(meta)
-        mean_reward, std_reward = parse_rewards(accuracy)
-        mean_reward = mean_reward if not pd.isna(mean_reward) else 0
-        std_reward = std_reward if not pd.isna(std_reward) else 0
-        row["Results"] = mean_reward - std_reward
-        row["Mean Reward"] = mean_reward
-        row["Std Reward"] = std_reward
-        data.append(row)
-
-    ranked_dataframe = rank_dataframe(pd.DataFrame.from_records(data))
-    new_history = ranked_dataframe
-    file_path = path + "/" + rl_env + ".csv"
-    new_history.to_csv(file_path, index=False)
-
-    return ranked_dataframe
-
-def download_leaderboard_dataset():
-    path = snapshot_download(repo_id=DATASET_REPO_ID, repo_type="dataset")
-    return path
-
-def get_data(rl_env, path) -> pd.DataFrame:
-    """
-    Get data from rl_env
-    :return: data as a pandas DataFrame
-    """
-    csv_path = path + "/" + rl_env + ".csv"
-    data = pd.read_csv(csv_path)
-
-    for index, row in data.iterrows():
-        user_id = row["User"]
-        data.loc[index, "User"] = make_clickable_user(user_id)
-        model_id = row["Model"]
-        data.loc[index, "Model"] = make_clickable_model(model_id)
-        
-    return data
-
-def get_data_no_html(rl_env, path) -> pd.DataFrame:
-    """
-    Get data from rl_env
-    :return: data as a pandas DataFrame
-    """
-    csv_path = path + "/" + rl_env + ".csv"
-    data = pd.read_csv(csv_path)
+    
+    for submission in submissions:
+        parsed_data = parse_submission_data(submission)
+        if parsed_data:
+            data.append(parsed_data)
+    
+    if not data:
+        # Create empty dataframe with correct columns if no submissions
+        return pd.DataFrame(columns=["User", "Model", "Dataset", "Hit Rate Click@50", "Hit Rate A2C@50", "Hit Rate Purchase@50", "Comment"])
+    
+    df = pd.DataFrame(data)
+    # Sort by hit rate click@50 (descending) as default
+    df = df.sort_values(by='Hit Rate Click@50', ascending=False)
+    df.reset_index(drop=True, inplace=True)
+    df.insert(0, 'Ranking', range(1, len(df) + 1))
+    
+    return df
 
-    return data
     
 def rank_dataframe(dataframe):
     dataframe = dataframe.sort_values(by=['Results', 'User', 'Model'], ascending=False)
@@ -305,108 +123,72 @@ def rank_dataframe(dataframe):
         dataframe['Ranking'] =   [i for i in range(1,len(dataframe)+1)]
     return dataframe
 
+def get_leaderboard_data():
+    """Get current leaderboard data from submissions"""
+    return update_leaderboard_from_submissions()
 
-def run_update_dataset():
-    path_ = download_leaderboard_dataset()
-    for i in range(0, len(rl_envs)):
-        rl_env = rl_envs[i]
-        update_leaderboard_dataset_parallel(rl_env["rl_env"], path_)
-
-    api.upload_folder(
-    folder_path=path_,
-    repo_id="huggingface-projects/drlc-leaderboard-data",
-    repo_type="dataset",
-    commit_message="Update dataset")
-
-def filter_data(rl_env, path, user_id):
-    data_df = get_data_no_html(rl_env, path)
-    models = []
-    models = data_df[data_df["User"] == user_id]
-
-    for index, row in models.iterrows():
-        user_id = row["User"]
-        models.loc[index, "User"] = make_clickable_user(user_id)
-        model_id = row["Model"]
-        models.loc[index, "Model"] = make_clickable_model(model_id)
-        
-
-    return models
+def refresh_leaderboard():
+    """Simple function to refresh the leaderboard display"""
+    print("🔄 Refreshing leaderboard...")
+    return get_leaderboard_data()
 
-run_update_dataset()
+# run_update_dataset()
 
 with block:
-    gr.Markdown(f"""
-    # 🏆 The Deep Reinforcement Learning Course Leaderboard 🏆 
+    gr.Markdown("""
+    # 🏆 Retrieval Task Leaderboard 🏆 
     
-    This is the leaderboard of trained agents during the <a href="https://huggingface.co/learn/deep-rl-course/unit0/introduction?fw=pt">Deep Reinforcement Learning Course</a>. A free course from beginner to expert.
+    This leaderboard tracks the performance of different models on retrieval tasks.
     
-    ### We only display the best 100 models
-    If you want to **find yours, type your user id and click on Search my models.**
-    You **can click on the model's name** to be redirected to its model card, including documentation.
+    ### How to Submit
+    Submit your results as a JSON file in the `submission` folder via pull request.
     
-    ### How are the results calculated?
-    We use **lower bound result to sort the models: mean_reward - std_reward.**
-
-    ### I can't find my model 😭
-    The leaderboard is **updated every two hours** if you can't find your models, just wait for the next update.
+    ### Required JSON Format
+    ```json
+    {
+        "user_id": "your_username",
+        "model_id": "your_model_name", 
+        "hit_rate_click@50": "0.75",
+        "hit_rate_A2C@50": "0.68",
+        "hit_rate_purchase@50": "0.82",
+        "dataset_id": "your_dataset",
+        "comment": "Optional comment about your submission"
+    }
+    ```
     
-    ### The Deep RL Course
-    🤖 You want to try to train your agents? <a href="https://huggingface.co/deep-rl-course/unit0/introduction?fw=pt" target="_blank"> Check the Hugging Face free Deep Reinforcement Learning Course 🤗 </a>.
-        
-    🔧 There is an **environment missing?** Please open an issue.
+    ### How to Update After PR
+    **Currently, PR detection is NOT automated.** After a PR is merged:
+    1. Restart the application manually: `python3 app.py`
+    2. Or click the "🔄 Refresh Leaderboard" button below
+    
+    ### Rankings
+    Currently ranked by "Hit Rate Click@50" (you can modify the sorting in the code)
     """)
-    path_ = download_leaderboard_dataset()
-
-    for i in range(0, len(rl_envs)):
-        rl_env = rl_envs[i]
-        with gr.TabItem(rl_env["rl_env_beautiful"]) as rl_tab:
-            with gr.Row():
-                markdown = """
-                    # {name_leaderboard}
-                    
-                    """.format(name_leaderboard = rl_env["rl_env_beautiful"], video_link = rl_env["video_link"])
-                gr.Markdown(markdown)
-                
-            
-            with gr.Row():
-                gr.Markdown("""
-                    ## Search your models
-                    Simply type your user id to find your models
-                    """)
-                
-            with gr.Row():
-                user_id = gr.Textbox(label= "Your user id")
-                search_btn = gr.Button("Search my models 🔎")
-                reset_btn = gr.Button("Clear my search")
-                env = gr.State(rl_env["rl_env"])
-                grpath = gr.State(path_)
-            with gr.Row():
-                gr_dataframe = gr.components.Dataframe(value=get_data(rl_env["rl_env"], path_), headers=["Ranking 🏆", "User 🤗", "Model id 🤖", "Results", "Mean Reward", "Std Reward"], datatype=["number", "markdown", "markdown", "number", "number", "number"],  row_count=(100, 'fixed'))
-             
-            with gr.Row():
-                #gr_search_dataframe = gr.components.Dataframe(headers=["Ranking 🏆", "User 🤗", "Model id 🤖", "Results", "Mean Reward", "Std Reward"], datatype=["number", "markdown", "markdown", "number", "number", "number"], visible=False)
-                search_btn.click(fn=filter_data, inputs=[env, grpath, user_id], outputs=gr_dataframe, api_name="filter_data")
-
-            with gr.Row():
-                search_btn.click(fn=filter_data, inputs=[env, grpath, user_id], outputs=gr_dataframe, api_name="filter_data")
-                reset_btn.click(fn=get_data, inputs=[env, grpath], outputs=gr_dataframe, api_name="get_data")
-    """
-    block.load(
-        download_leaderboard_dataset,
-        inputs=[],
-        outputs=[
-            grpath
-        ],
+    
+    # Simple refresh button
+    refresh_button = gr.Button("🔄 Refresh Leaderboard")
+    
+    # Display leaderboard without Overall Score column
+    leaderboard_df = gr.Dataframe(
+        value=get_leaderboard_data(),
+        headers=["Ranking", "User", "Model", "Dataset", "Hit Rate Click@50", "Hit Rate A2C@50", "Hit Rate Purchase@50", "Comment"],
+        label="Current Leaderboard"
     )
-    """
+    
+    # Simple refresh functionality
+    refresh_button.click(refresh_leaderboard, outputs=leaderboard_df)
 
+# Initialize the system
+print("🚀 Starting Retrieval Task Leaderboard...")
 
+# Setup background scheduler (optional, mainly for restart)
 scheduler = BackgroundScheduler()
-# Refresh every hour
-#scheduler.add_job(func=run_update_dataset, trigger="interval", seconds=3600)
-#scheduler.add_job(download_leaderboard_dataset, 'interval', seconds=3600)
-#scheduler.add_job(run_update_dataset, 'interval', seconds=3600)
-scheduler.add_job(restart, 'interval', seconds=10800)
+scheduler.add_job(restart, 'interval', seconds=21600)  # Restart every 6 hours
 scheduler.start()
 
+print("✅ System initialized successfully!")
+print("📊 Leaderboard accessible at: http://127.0.0.1:7860")
+print("⚠️  PR detection is NOT automated - restart manually after PR merges")
+print("🔄 Use the refresh button in the UI to update the leaderboard")
+
 block.launch()

requirements.txt

@@ -1,4 +1,7 @@
 APScheduler==3.10.1
-gradio==4.0
-httpx==0.24.0
-tqdm
+gradio==4.44.0
+fastapi==0.112.2
+pydantic==2.10.6
+httpx>=0.24.1
+tqdm
+huggingface-hub>=0.19.3

submission/a_random_submit.json

@@ -0,0 +1,9 @@
+{
+    "user_id": "a_random_submit",
+    "model_id": "a_random_submit",
+    "hit_rate_click@50": "0",
+    "hit_rate_A2C@50": "0",
+    "hit_rate_purchase@50": "0",
+    "dataset_id": "a_random_submit",
+    "comment": "a_random_submit"
+}

submission/test_submission_1.json

@@ -0,0 +1,9 @@
+{
+    "user_id": "alice_researcher",
+    "model_id": "bert-retrieval-v1",
+    "hit_rate_click@50": "0.85",
+    "hit_rate_A2C@50": "0.78",
+    "hit_rate_purchase@50": "0.92",
+    "dataset_id": "ecommerce-dataset-v1",
+    "comment": "BERT-based retrieval model with fine-tuning"
+} 

submission/test_submission_2.json

@@ -0,0 +1,9 @@
+{
+    "user_id": "bob_nlp",
+    "model_id": "transformer-retrieval-v2",
+    "hit_rate_click@50": "0.82",
+    "hit_rate_A2C@50": "0.89",
+    "hit_rate_purchase@50": "0.87",
+    "dataset_id": "retail-dataset-v2",
+    "comment": "Transformer-based approach with attention mechanisms"
+}