Update app.py

app.py

@@ -1,58 +1,15 @@
-import os
-os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
 from openllm import *
 import requests
 import pandas as pd
 from bs4 import BeautifulSoup
 from tqdm import tqdm
-from huggingface_hub import HfApi, CommitOperationAdd, create_commit
+from huggingface_hub import HfApi
 import gradio as gr
 import datetime
 
 api = HfApi()
 
 
-HF_TOKEN = os.getenv('HF_TOKEN')
-
-
-headers_models = ["🔢 Serial Number", "👤 Author Name", "📥 Total Downloads", "👍 Total Likes", "🤖 Number of Models",
-            "🏆 Best Model On Open LLM Leaderboard", "🥇 Best Rank On Open LLM Leaderboard",
-            "📊 Average Downloads per Model", "📈 Average Likes per Model", "🚀 Most Downloaded Model",
-            "📈 Most Download Count", "❤️ Most Liked Model", "👍 Most Like Count", "🔥 Trending Model",
-            "👑 Best Rank at Trending Models", "🏷️ Type"]
-
-headers_datasets = ["🔢 Serial Number", "👤 Author Name", "📥 Total Downloads", "👍 Total Likes", "📊 Number of Datasets",
-            "📊 Average Downloads per Dataset", "📈 Average Likes per Dataset", "🚀 Most Downloaded Dataset",
-            "📈 Most Download Count", "❤️ Most Liked Dataset", "👍 Most Like Count", "🔥 Trending Dataset",
-            "👑 Best Rank at Trending Datasets", "🏷️ Type"]
-
-headers_spaces = ["🔢 Serial Number", "👤 Author Name", "👍 Total Likes", "🚀 Number of Spaces", "📈 Average Likes per Space",
-            "❤️ Most Liked Space", "👍 Most Like Count", "🔥 Trending Space", "👑 Best Rank at Trending Spaces",
-            "🏷️ Type"]
-
-
-def apply_headers(df, headers):
-    tmp = df.copy()
-    tmp.columns = headers
-
-    return tmp
-
-
-def get_time():
-    return datetime.datetime.now().strftime("%d-%m-%Y %H-%M")
-
-
-def upload_datasets(dfs):
-
-  time = get_time()
-
-  operations = [CommitOperationAdd(path_in_repo=f"{time}/models_df.csv", path_or_fileobj=(dfs[0].to_csv()).encode()),
-                CommitOperationAdd(path_in_repo=f"{time}/datasets_df.csv", path_or_fileobj=(dfs[1].to_csv()).encode()),
-                CommitOperationAdd(path_in_repo=f"{time}/spaces_df.csv", path_or_fileobj=(dfs[2].to_csv()).encode())]
-
-  return (create_commit(repo_id="Weyaxi/huggingface-leaderboard-history", operations=operations, commit_message=f"Uploading history of {time}", repo_type="dataset", token=HF_TOKEN))
-
-
 def get_most(df_for_most_function):
     download_sorted_df = df_for_most_function.sort_values(by=['downloads'], ascending=False)
     most_downloaded = download_sorted_df.iloc[0]
@@ -73,14 +30,10 @@ def get_sum(df_for_sum_function):
 
 
 def get_openllm_leaderboard():
-    try:
-        data = get_json_format_data()
-        finished_models = get_datas(data)
-        df = pd.DataFrame(finished_models)
-        return df['Model'].tolist()
-    except Exception as e:  # something is wrong about the leaderboard so return empty list
-        print(e)
-        return []
+    data = get_json_format_data()
+    finished_models = get_datas(data)
+    df = pd.DataFrame(finished_models)
+    return df['Model'].tolist()
 
 
 def get_ranking(model_list, target_org):
@@ -259,21 +212,21 @@ def get_ranking_trend(json_data, org_name):
         return {"id": "Not Found", "rank": "Not Found"}
 
 
-
 def fetch_data_from_url(url):
     response = requests.get(url)
     if response.status_code == 200:
         data = response.text.splitlines()
-        return [line.rstrip("\n") for line in data]
+        return [line.strip() for line in data]
     else:
         print(f"Failed to fetch data from URL: {url}")
         return []
 
-user_names_url = "https://huggingface.co/datasets/Weyaxi/user-orgs-huggingface-leaderboard/raw/main/user_names.txt"
-org_names_url = "https://huggingface.co/datasets/Weyaxi/user-orgs-huggingface-leaderboard/raw/main/org_names.txt"
 
-user_names_in_list = fetch_data_from_url(user_names_url)
-org_names_in_list = fetch_data_from_url(org_names_url)
+user_names_url = "https://huggingface.co/datasets/PulsarAI/user-orgs-huggingface-leaderboard/raw/main/user_names.txt"
+org_names_url = "https://huggingface.co/datasets/PulsarAI/user-orgs-huggingface-leaderboard/raw/main/org_names.txt"
+
+org_names_in_list = fetch_data_from_url(user_names_url)
+user_names_in_list = fetch_data_from_url(org_names_url)
 
 datetime_now = str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
 INTRODUCTION_TEXT = f"""
@@ -291,9 +244,9 @@ INTRODUCTION_TEXT = f"""
 
 🛠️ The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
 
-📒 **Note:** In the model's dataframe, there are some columns related to the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). This data is also retrieved through web scraping.
+**📝 Note:** In the model's dataframe, there are some columns related to the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). This data is also retrieved through web scraping.
 
-📒 **Note:** In trending models/datasets/spaces, first 300 models/datasets/spaces is being retrieved from huggingface.
+**📝 Note:** In trending models/datasets/spaces, first 300 models/datasets/spaces is being retrieved from huggingface.
 
 ## 🔍 Searching Organizations and Users
 
@@ -346,12 +299,12 @@ def update_table(orgs, users, how_much=400, return_all=False):
         filtered_df = dataFrame[(dataFrame['Type'] == 'Organization') | (dataFrame['Type'] == 'User')]
 
     else:
-        return apply_headers(dataFrame.head(0), headers_models)
+        return dataFrame.head(0)
 
     if return_all:
-        return apply_headers(filtered_df, headers_models)
+        return filtered_df
     else:
-        return apply_headers(filtered_df, headers_models).head(how_much)
+        return filtered_df.head(how_much)
 
 
 def update_table_datasets(orgs, users, how_much=250, return_all=False):
@@ -367,12 +320,12 @@ def update_table_datasets(orgs, users, how_much=250, return_all=False):
         filtered_df = dataFrame[(dataFrame['Type'] == 'Organization') | (dataFrame['Type'] == 'User')]
 
     else:
-        return apply_headers(dataFrame, headers_datasets).head(0)
+        return dataFrame.head(0)
 
     if return_all:
-        return apply_headers(filtered_df, headers_datasets)
+        return filtered_df
     else:
-        return apply_headers(filtered_df, headers_datasets).head(how_much)
+        return filtered_df.head(how_much)
 
 
 def update_table_spaces(orgs, users, how_much=200, return_all=False):
@@ -388,12 +341,12 @@ def update_table_spaces(orgs, users, how_much=200, return_all=False):
         filtered_df = dataFrame[(dataFrame['Type'] == 'Organization') | (dataFrame['Type'] == 'User')]
 
     else:
-        return apply_headers(dataFrame, headers_spaces).head(0)
+        return dataFrame.head(0)
 
     if return_all:
-        return apply_headers(filtered_df, headers_spaces)
+        return filtered_df
     else:
-        return apply_headers(filtered_df, headers_spaces).head(how_much)
+        return filtered_df.head(how_much)
 
 
 
@@ -503,6 +456,7 @@ def search_df(author):
   return markdown_text
 
 
+
 with gr.Blocks() as demo:
     gr.Markdown("""<h1 align="center" id="space-title">🤗 Huggingface Leaderboard</h1>""")
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -522,7 +476,13 @@ with gr.Blocks() as demo:
         models_df = make_leaderboard(org_names_in_list, user_names_in_list, "models", group_models_by_author(all_models))
         models_df = models_df_to_clickable(models_df, columns_to_convert, "models")
 
-        gr_models = gr.Dataframe(apply_headers(models_df, headers_models).head(400), headers=headers_models, interactive=True,
+        headers = ["🔢 Serial Number", "👤 Author Name", "📥 Total Downloads", "👍 Total Likes", "🤖 Number of Models",
+                   "🏆 Best Model On Open LLM Leaderboard", "🥇 Best Rank On Open LLM Leaderboard",
+                   "📊 Average Downloads per Model", "📈 Average Likes per Model", "🚀 Most Downloaded Model",
+                   "📈 Most Download Count", "❤️ Most Liked Model", "👍 Most Like Count", "🔥 Trending Model",
+                   "👑 Best Rank at Trending Models", "🏷️ Type"]
+
+        gr_models = gr.Dataframe(models_df.head(400), headers=headers, interactive=True,
                                  datatype=["str", "markdown", "str", "str", "str", "markdown", "str", "str", "str",
                                            "markdown", "str", "markdown", "str", "markdown", "str", "str"])
 
@@ -531,7 +491,12 @@ with gr.Blocks() as demo:
         dataset_df = make_leaderboard(org_names_in_list, user_names_in_list, "datasets", group_models_by_author(all_datasets))
         dataset_df = models_df_to_clickable(dataset_df, columns_to_convert, "datasets")
 
-        gr_datasets = gr.Dataframe(apply_headers(dataset_df, headers_datasets).head(250), headers=headers_datasets, interactive=False,
+        headers = ["🔢 Serial Number", "👤 Author Name", "📥 Total Downloads", "👍 Total Likes", "📊 Number of Datasets",
+                   "📊 Average Downloads per Dataset", "📈 Average Likes per Dataset", "🚀 Most Downloaded Dataset",
+                   "📈 Most Download Count", "❤️ Most Liked Dataset", "👍 Most Like Count", "🔥 Trending Dataset",
+                   "👑 Best Rank at Trending Datasets", "🏷️ Type"]
+
+        gr_datasets = gr.Dataframe(dataset_df.head(250), headers=headers, interactive=False,
                                    datatype=["str", "markdown", "str", "str", "str", "str", "str", "markdown", "str",
                                              "markdown", "str", "markdown", "str", "str"])
 
@@ -541,11 +506,14 @@ with gr.Blocks() as demo:
         spaces_df = make_leaderboard(org_names_in_list, user_names_in_list, "spaces", group_models_by_author(all_spaces))
         spaces_df = models_df_to_clickable(spaces_df, columns_to_convert, "spaces")
 
-        gr_spaces = gr.Dataframe(apply_headers(spaces_df, headers_spaces).head(200), headers=headers_spaces, interactive=False,
+        headers = ["🔢 Serial Number", "👤 Author Name", "👍 Total Likes", "🚀 Number of Spaces", "📈 Average Likes per Space",
+                   "❤️ Most Liked Space", "👍 Most Like Count", "🔥 Trending Space", "👑 Best Rank at Trending Spaces",
+                   "🏷️ Type"]
+
+        gr_spaces = gr.Dataframe(spaces_df.head(200), headers=headers, interactive=False,
                                  datatype=["str", "markdown", "str", "str", "str", "markdown", "str", "markdown", "str",
                                            "str"])
 
-    
     with gr.TabItem("🔍 Search", id=4):
       with gr.Column(min_width=320):
             search_bar = gr.Textbox(
@@ -557,9 +525,6 @@ with gr.Blocks() as demo:
       search_bar.submit(fn=search_df, inputs=search_bar, outputs=yazi)
 
 
-    commit = upload_datasets([models_df, dataset_df, spaces_df])
-    print(commit)
-
     orgs.change(fn=update_table, inputs=[orgs, users], outputs=gr_models)
 
     orgs.change(fn=update_table_datasets, inputs=[orgs, users], outputs=gr_datasets)
@@ -573,14 +538,13 @@ with gr.Blocks() as demo:
     users.change(fn=update_table_spaces, inputs=[orgs, users], outputs=gr_spaces)
 
 
-filtered_model_users = update_table(orgs=False, users=True, return_all=True)['👤 Author Name'].tolist()
-filtered_model_orgs = update_table(orgs=True, users=False, return_all=True)['👤 Author Name'].tolist()
+filtered_model_users = update_table(orgs=False, users=True, return_all=True)['Author Name'].tolist()
+filtered_model_orgs = update_table(orgs=True, users=False, return_all=True)['Author Name'].tolist()
 
-filtered_datasets_users = update_table_datasets(orgs=False, users=True, return_all=True)['👤 Author Name'].tolist()
-filtered_datasets_orgs = update_table_datasets(orgs=True, users=False, return_all=True)['👤 Author Name'].tolist()
+filtered_datasets_users = update_table_datasets(orgs=False, users=True, return_all=True)['Author Name'].tolist()
+filtered_datasets_orgs = update_table_datasets(orgs=True, users=False, return_all=True)['Author Name'].tolist()
 
-filtered_spaces_users = update_table_spaces(orgs=False, users=True, return_all=True)['👤 Author Name'].tolist()
-filtered_spaces_orgs = update_table_spaces(orgs=True, users=False, return_all=True)['👤 Author Name'].tolist()
+filtered_spaces_users = update_table_spaces(orgs=False, users=True, return_all=True)['Author Name'].tolist()
+filtered_spaces_orgs = update_table_spaces(orgs=True, users=False, return_all=True)['Author Name'].tolist()
 
 demo.launch(debug=True)
-