Strange system with file for persistent check

app.py

@@ -16,7 +16,16 @@ from src.display.utils import (
     AutoEvalColumn,
     fields,
 )
-from src.envs import API, H4_TOKEN, HF_HOME, REPO_ID, RESET_JUDGEMENT_ENV
+from src.envs import (
+    API,
+    H4_TOKEN,
+    HF_HOME,
+    METAINFO_DATASET,
+    PERSISTENT_FILE_CHECK,
+    PERSISTENT_FILE_CHECK_PATH,
+    REPO_ID,
+    RESET_JUDGEMENT_ENV,
+)
 from src.leaderboard.build_leaderboard import build_leadearboard_df, download_openbench
 
 os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"
@@ -30,10 +39,6 @@ enable_space_ci()
 download_openbench()
 
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
-
-
 def build_demo():
     demo = gr.Blocks(title="Chatbot Arena Leaderboard", css=custom_css)
     leaderboard_df = build_leadearboard_df()
@@ -80,6 +85,14 @@ def build_demo():
                             repo_id="Vikhrmodels/openbench-eval",
                             repo_type="dataset",
                         )
+                        with open(PERSISTENT_FILE_CHECK_PATH, "w", encoding="utf-8") as f:
+                            f.write("1")
+                        API.upload_file(
+                            path_or_fileobj=PERSISTENT_FILE_CHECK,
+                            path_in_repo="",
+                            repo_id=METAINFO_DATASET,
+                            repo_type="dataset",
+                        )
                         os.environ[RESET_JUDGEMENT_ENV] = "1"
                         return file.name
 
@@ -98,24 +111,42 @@ def build_demo():
 
 
 def update_board():
+    # very shitty solution, where we update board only when needed
+    # the state is checked by the the file PERSISTENT_FILE_CHECK
+    # very bad solution
+    # but a fast one to code
     need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
     logging.info("Updating the judgement: %s", need_reset)
     if need_reset != "1":
         return
     os.environ[RESET_JUDGEMENT_ENV] = "0"
-
-    # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py")
-    # subprocess.run(["python3", gen_judgement_file], check=True)
-
-    show_result_file = os.path.join(HF_HOME, "src/gen/show_result.py")
-    subprocess.run(["python3", show_result_file, "--output"], check=True)
-
-    # update the gr item with leaderboard
-    # TODO
+    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
 
 if __name__ == "__main__":
-    os.environ[RESET_JUDGEMENT_ENV] = "1"
+    os.environ[RESET_JUDGEMENT_ENV] = "0"
+
+    need_recalc = False
+    try:
+        with open(PERSISTENT_FILE_CHECK_PATH, "r", encoding="utf-8") as f:
+            need_recalc = f.read() == "1"
+        with open(PERSISTENT_FILE_CHECK_PATH, "w", encoding="utf-8") as f:
+            f.write("0")
+    except FileNotFoundError:
+        pass
+
+    if need_recalc:
+        API.upload_file(
+            path_or_fileobj=PERSISTENT_FILE_CHECK,
+            path_in_repo="",
+            repo_id=METAINFO_DATASET,
+            repo_type="dataset",
+        )
+        # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py")
+        # subprocess.run(["python3", gen_judgement_file], check=True)
+
+        show_result_file = os.path.join(HF_HOME, "src/gen/show_result.py")
+        subprocess.run(["python3", show_result_file, "--output"], check=True)
 
     scheduler = BackgroundScheduler()
     scheduler.add_job(update_board, "interval", minutes=10)

src/envs.py

@@ -35,6 +35,11 @@ RESET_JUDGEMENT_ENV = "RESET_JUDGEMENT"
 
 API = HfApi(token=H4_TOKEN)
 
+PERSISTENT_FILE_CHECK = "persistent_file_check"
+PERSISTENT_FILE_CHECK_PATH = f"{DATA_PATH}/{PERSISTENT_FILE_CHECK}"
+
+METAINFO_DATASET = "Vikhrmodels/arena-leaderboard-metainfo"
+
 # useless env
 EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "data/eval-queue")
 PATH_TO_COLLECTION = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03"

src/gen/show_result.py

@@ -12,7 +12,7 @@ from sklearn.linear_model import LogisticRegression
 from tqdm import tqdm
 from utils import load_model_answers
 
-from src.envs import HF_TOKEN_PRIVATE
+from src.envs import HF_TOKEN_PRIVATE, METAINFO_DATASET
 
 
 def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
@@ -265,7 +265,7 @@ if __name__ == "__main__":
         huggingface_hub.HfApi().upload_file(
             path_or_fileobj=json_file_name,
             path_in_repo="leaderboard.json",
-            repo_id="Vikhrmodels/arena-leaderboard-metainfo",
+            repo_id=METAINFO_DATASET,
             repo_type="dataset",
             token=HF_TOKEN_PRIVATE,
         )
@@ -273,7 +273,7 @@ if __name__ == "__main__":
         huggingface_hub.HfApi().upload_file(
             path_or_fileobj=json_file_name,
             path_in_repo=f"leaderboard_logs/{json_file_name}",
-            repo_id="Vikhrmodels/arena-leaderboard-metainfo",
+            repo_id=METAINFO_DATASET,
             repo_type="dataset",
             token=HF_TOKEN_PRIVATE,
         )

src/leaderboard/build_leaderboard.py

@@ -6,7 +6,7 @@ import time
 import pandas as pd
 from huggingface_hub import snapshot_download
 
-from src.envs import DATA_ARENA_PATH, DATA_PATH, HF_TOKEN_PRIVATE
+from src.envs import DATA_ARENA_PATH, DATA_PATH, HF_TOKEN_PRIVATE, METAINFO_DATASET
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -53,7 +53,7 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
 
 def download_openbench():
     # download prev autogenerated leaderboard files
-    download_dataset("Vikhrmodels/arena-leaderboard-metainfo", DATA_PATH)
+    download_dataset(METAINFO_DATASET, DATA_PATH)
 
     # download answers of different models that we trust
     download_dataset("Vikhrmodels/openbench-eval", DATA_ARENA_PATH)