Update to new format

src/__init__.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+from typing import TypeAlias
+from zoneinfo import ZoneInfo
+
+TIMEZONE = ZoneInfo("America/Los_Angeles")
+START_DATE = datetime(2024, 11, 9)
+
+Uid: TypeAlias = int
+Hotkey: TypeAlias = str

src/app.py

@@ -2,16 +2,16 @@ import gradio as gr
 
 from chain_data import sync_metagraph
 from leaderboard import create_leaderboard, create_dropdown
+from model_demo import create_demo
 from submissions import create_submissions
 from validator_states import create_validator_states
 from validator_weights import create_weights
-from model_demo import create_demo
 from wandb_data import sync
 
 
 def main():
-    sync_metagraph(timeout=1000)
-    sync(timeout=1000)
+    sync_metagraph()
+    sync()
     with gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}", fill_height=True, fill_width=True) as app:
         with gr.Tab("Leaderboard") as leaderboard_tab:
             dropdown = gr.Dropdown()

src/chain_data.py

@@ -1,4 +1,4 @@
-from concurrent.futures import ThreadPoolExecutor
+import os
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import Enum
@@ -12,15 +12,19 @@ from fiber.chain.models import Node
 from substrateinterface.storage import StorageKey
 
 from network_commitments import Decoder
-from wandb_data import TIMEZONE, Hotkey, Uid
+from src import Hotkey, Uid, TIMEZONE
+
+DISABLE_COMMITMENTS_FETCH = int(os.getenv("DISABLE_COMMITMENTS_FETCH") or 0) > 0
 
 Weight: TypeAlias = float
 Incentive: TypeAlias = float
 
+
 class ContestId(Enum):
     FLUX_NVIDIA_4090 = 0
     SDXL_NEWDREAM_NVIDIA_4090 = 1
 
+
 @dataclass
 class Commitment:
     provider: str
@@ -47,6 +51,7 @@ class Commitment:
     def get_repo_link(self):
         return f"https://{self.provider}/{self.repository}"
 
+
 SPEC_VERSION = 7
 NET_UID = 39
 WEIGHTS_BY_MINER: dict[Hotkey, list[tuple[Hotkey, Weight]]] = {}
@@ -128,11 +133,15 @@ def fetch_identities(block: int):
     for hotkey, node in metagraph.nodes.items():
         for storage, info in identities:
             if node.coldkey != storage.params[0]: continue
-            if info != None: # noqa
+            if info != None:  # noqa
                 VALIDATOR_IDENTITIES[hotkey] = info.value["name"]
                 break
 
+
 def fetch_commitments(block: int):
+    if DISABLE_COMMITMENTS_FETCH:
+        return
+
     COMMITMENTS.clear()
     storage_keys: list[StorageKey] = []
     for hotkey, node in metagraph.nodes.items():
@@ -172,7 +181,7 @@ last_identity_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 last_commitment_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
-def sync_metagraph(timeout: int = 10):
+def sync_metagraph():
     global substrate
     global last_sync
     now = datetime.now(TIMEZONE)
@@ -180,7 +189,7 @@ def sync_metagraph(timeout: int = 10):
         return
     last_sync = now
 
-    def sync_task():
+    try:
         print("Syncing metagraph...")
         block = substrate.get_block_number(None)  # type: ignore
         metagraph.sync_nodes()
@@ -203,13 +212,6 @@ def sync_metagraph(timeout: int = 10):
             print("Syncing commitments...")
             last_commitment_sync = now
             fetch_commitments(block)
-
-    with ThreadPoolExecutor(max_workers=1) as executor:
-        future = executor.submit(sync_task)
-        try:
-            future.result(timeout=timeout)
-        except TimeoutError:
-            print("Timed out while syncing metagraph")
-        except Exception as e:
-            print(f"Error occurred while syncing metagraph: {e}")
-            substrate = get_substrate()
+    except Exception as e:
+        print(f"Error occurred while syncing metagraph: {e}")
+        substrate = get_substrate()

src/leaderboard.py

@@ -7,6 +7,7 @@ from wandb_data import get_current_runs
 
 DEFAULT_VALIDATOR_UID = int(os.environ["DEFAULT_VALIDATOR_UID"])
 
+
 def create_dropdown() -> gr.Dropdown:
     choices: list[tuple[str, int]] = []
     runs = get_current_runs()
@@ -26,17 +27,18 @@ def create_dropdown() -> gr.Dropdown:
         label="Source Validator"
     )
 
+
 def create_leaderboard(validator_uid) -> gr.Dataframe:
     data: list[list] = []
     runs = get_current_runs()
     for run in runs:
         if run.uid != validator_uid:
             continue
-        for submission in run.submissions.values():
+        for hotkey, submission in run.submissions.items():
             data.append([
                 submission.info.uid,
                 f"[{'/'.join(submission.info.repository.split('/')[-2:])}]({submission.info.repository})",
-                submission.tier,
+                submission.rank + 1,
                 round(submission.score, 3),
                 f"{submission.metrics.generation_time:.3f}s",
                 f"{submission.average_similarity * 100:.3f}%",
@@ -46,13 +48,13 @@ def create_leaderboard(validator_uid) -> gr.Dataframe:
                 f"{submission.metrics.load_time:.3f}s",
                 f"[{submission.info.block}](https://taostats.io/block/{submission.info.block})",
                 f"[{submission.info.revision}]({submission.info.repository}/commit/{submission.info.revision})",
-                f"[{submission.info.hotkey[:6]}...](https://taostats.io/hotkey/{submission.info.hotkey})",
+                f"[{hotkey[:6]}...](https://taostats.io/hotkey/{hotkey})",
             ])
 
-    data.sort(key=lambda x: (-x[2], int(x[10].split('[')[1].split(']')[0])))
+    data.sort(key=lambda x: (x[2], int(x[10].split('[')[1].split(']')[0])))
 
     return gr.Dataframe(
-        pd.DataFrame(data, columns=["UID", "Model", "Tier", "Score", "Gen Time", "Similarity", "Size", "VRAM Usage", "Power Usage", "Load Time", "Block", "Revision", "Hotkey"]),
+        pd.DataFrame(data, columns=["UID", "Model", "Rank", "Score", "Gen Time", "Similarity", "Size", "VRAM Usage", "Power Usage", "Load Time", "Block", "Revision", "Hotkey"]),
         datatype=["number", "markdown", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown", "markdown"],
         interactive=False,
         max_height=800,

src/model_demo.py

@@ -17,6 +17,7 @@ SERVER_API_KEY = os.environ["SERVER_API_KEY"]
 current_model: str | None = None
 last_current_model_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
+
 def get_current_model() -> str | None:
     global current_model
     global last_current_model_sync
@@ -36,7 +37,6 @@ def get_current_model() -> str | None:
         return None
 
 
-
 def image_from_base64(image_data: str) -> Image:
     image_buffer = BytesIO(base64.b64decode(image_data))
     image = Image.open(image_buffer)

src/network_commitments.py

@@ -1,6 +1,5 @@
 from struct import pack, unpack
 
-
 _UINT_16_SIZE = 2
 _UINT_32_SIZE = 4
 

src/submissions.py

@@ -2,18 +2,22 @@ import gradio as gr
 import pandas as pd
 
 from chain_data import sync_metagraph, COMMITMENTS, UIDS_BY_HOTKEY
-from wandb_data import get_current_runs, Run, Uid
+from src import Hotkey
+from wandb_data import get_current_runs, Run
 
-def get_status(run: Run, uid: Uid, block: int) -> tuple[str, str]:
-    if all(not submission.get(uid) or block > submission[uid].info.block for submission in [run.submissions, run.invalid_submissions]):
+
+def get_status(run: Run, hotkey: Hotkey, block: int) -> tuple[str, str]:
+    if hotkey in run.submissions and block > run.submissions[hotkey].info.block and hotkey not in run.invalid_submissions:
         return "Pending", "orange"
 
-    if uid in run.submissions:
+    if hotkey in run.submissions:
         return "Done", "springgreen"
-    elif uid in run.invalid_submissions:
+
+    if hotkey in run.invalid_submissions:
         return "Invalid", "red"
-    else:
-        return "Pending", "orange"
+
+    return "Pending", "orange"
+
 
 def create_submissions() -> gr.Dataframe:
     data: list[list] = []
@@ -21,9 +25,8 @@ def create_submissions() -> gr.Dataframe:
     runs = sorted(get_current_runs(), key=lambda run: run.uid)
 
     for hotkey, commitment in COMMITMENTS.items():
-        uid = UIDS_BY_HOTKEY[hotkey]
         row = [
-            uid,
+            UIDS_BY_HOTKEY[hotkey],
             f"[{'/'.join(commitment.get_repo_link().split('/')[-2:])}]({commitment.get_repo_link()})",
             f"[{commitment.block}](https://taostats.io/block/{commitment.block})",
             f"[{commitment.revision}]({commitment.get_repo_link()}/commit/{commitment.revision})",
@@ -32,7 +35,7 @@ def create_submissions() -> gr.Dataframe:
         ]
 
         for run in runs:
-            status, color = get_status(run, uid, commitment.block)
+            status, color = get_status(run, hotkey, commitment.block)
             row.append(f"<span style='color: {color}'>{status}</span>")
 
         data.append(row)

src/validator_states.py

@@ -5,10 +5,10 @@ import gradio as gr
 import pandas as pd
 from packaging import version
 
-from wandb_data import get_current_runs, Run
 from chain_data import get_nodes, UPDATED
+from wandb_data import get_current_runs, Run, BenchmarkStatus
 
-AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 180  # 3 minutes
+AVERAGE_BENCHMARKING_TIME_WARNING_THRESHOLD = 180  # 3 minutes
 ETA_WARNING_THRESHOLD = 43200  # 12 hours
 UPDATED_WARNING_THRESHOLD = 1000
 VTRUST_WARNING_THRESHOLD = 0.75
@@ -30,7 +30,7 @@ def colorize(val, color: str) -> str:
 def create_validator_states() -> gr.Dataframe:
     data: list[list] = []
     runs = sorted(get_current_runs(), key=lambda run: run.uid)
-    winners = [run.winner_uid for run in runs if run.winner_uid]
+    winners = [run.winner_uid for run in runs if run.winner_uid and run.status == BenchmarkStatus.FINISHED]
     winner_uid_mode = statistics.mode(winners) if winners else None
     latest_version = get_latest_version(runs)
     for run in runs:
@@ -41,10 +41,10 @@ def create_validator_states() -> gr.Dataframe:
             run.name,
             colorize(run.version, "springgreen" if run.version == latest_version else "red"),
             colorize(run.status.name(), run.status.color()),
-            colorize(run.winner_uid, "springgreen" if winner_uid_mode and run.winner_uid == winner_uid_mode else "orange" if run.winner_uid else "gray"),
+            colorize(run.winner_uid, "springgreen" if winner_uid_mode and run.winner_uid == winner_uid_mode else "orange" if run.winner_uid and run.status == BenchmarkStatus.FINISHED else "gray"),
             f"{min(run.total_submissions, len(run.submissions) + len(run.invalid_submissions))}/{run.total_submissions}",
             len(run.invalid_submissions),
-            colorize(f"{timedelta(seconds=int(run.average_benchmark_time))}", "orange" if run.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else "springgreen" if run.average_benchmark_time > 0 else "gray"),
+            colorize(f"{timedelta(seconds=int(run.average_benchmarking_time))}", "orange" if run.average_benchmarking_time > AVERAGE_BENCHMARKING_TIME_WARNING_THRESHOLD else "springgreen" if run.average_benchmarking_time > 0 else "gray"),
             colorize(f"{timedelta(seconds=run.eta)}", "orange" if run.eta > ETA_WARNING_THRESHOLD else "springgreen" if run.eta > 0 else "gray"),
             colorize(f"{vtrust:.4f}", "springgreen" if vtrust > VTRUST_WARNING_THRESHOLD else "red"),
             colorize(updated, "springgreen" if updated < UPDATED_WARNING_THRESHOLD else "red"),

src/validator_weights.py

@@ -22,6 +22,7 @@ def get_color_by_weight(weight: float) -> str:
         g = int(255 - ((1 - progress) * 50))
         return f"rgb(0, {g}, 0)"
 
+
 def get_active_weights() -> dict[Hotkey, list[tuple[Hotkey, Weight]]]:
     runs = get_current_runs()
     weights: dict[Hotkey, list[tuple[Hotkey, Weight]]] = {}
@@ -35,6 +36,7 @@ def get_active_weights() -> dict[Hotkey, list[tuple[Hotkey, Weight]]]:
 
     return weights
 
+
 def create_weights(include_inactive: bool) -> gr.Dataframe:
     data: list[list] = []
     sync_metagraph()

src/wandb_data.py

@@ -1,22 +1,18 @@
 import os
-from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from enum import Enum
-from typing import TypeAlias
-from zoneinfo import ZoneInfo
 
 import wandb
 import wandb.apis.public as wapi
 from substrateinterface import Keypair
 
-WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
+from src import TIMEZONE, Hotkey
+from chain_data import UIDS_BY_HOTKEY, VALIDATOR_IDENTITIES, sync_metagraph
 
-TIMEZONE = ZoneInfo("America/Los_Angeles")
-START_DATE = datetime(2024, 11, 9)
+WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
 
-Uid: TypeAlias = int
-Hotkey: TypeAlias = str
+START_DATE = datetime(2024, 11, 29)
 
 
 class BenchmarkStatus(Enum):
@@ -51,7 +47,6 @@ class MetricData:
 @dataclass
 class SubmissionInfo:
     uid: int
-    hotkey: Hotkey
     repository: str
     revision: str
     block: int
@@ -63,16 +58,10 @@ class Submission:
     metrics: MetricData
     average_similarity: float
     min_similarity: float
-    tier: int
+    rank: int
     score: float
 
 
-@dataclass
-class InvalidSubmission:
-    info: SubmissionInfo
-    reason: str
-
-
 @dataclass
 class Run:
     start_date: datetime
@@ -81,29 +70,28 @@ class Run:
     name: str
     hotkey: str
     status: BenchmarkStatus
-    average_benchmark_time: float
+    average_benchmarking_time: float
     eta: int
     winner_uid: int | None
     baseline_metrics: MetricData | None
     total_submissions: int
-    submissions: dict[Uid, Submission]
-    invalid_submissions: dict[Uid, InvalidSubmission]
+    submissions: dict[Hotkey, Submission]
+    invalid_submissions: set[Hotkey]
 
 
 RUNS: dict[str, list[Run]] = {}
 
 
-def _is_valid_run(run: wapi.Run):
-    required_config_keys = ["hotkey", "uid", "contest", "signature"]
+def _is_valid_run(run: wapi.Run, version: str) -> bool:
+    required_config_keys = ["hotkey", "uid", "signature"]
 
     for key in required_config_keys:
         if key not in run.config:
             return False
 
     validator_hotkey = run.config["hotkey"]
-    contest_name = run.config["contest"]
 
-    signing_message = f"{run.name}:{validator_hotkey}:{contest_name}"
+    signing_message = f"{version}:{validator_hotkey}"
 
     return Keypair(validator_hotkey).verify(signing_message, run.config["signature"])
 
@@ -122,7 +110,8 @@ def _status_from_run(run: wapi.Run) -> BenchmarkStatus:
             return BenchmarkStatus.FAILED
         case "running":
             if "benchmarking_state" in run.summary:
-                return BenchmarkStatus[run.summary["benchmarking_state"]]
+                states = list(BenchmarkStatus)
+                return states[int(run.summary["benchmarking_state"])]
             else:
                 return BenchmarkStatus.INITIALIZING
         case _:
@@ -131,14 +120,15 @@ def _status_from_run(run: wapi.Run) -> BenchmarkStatus:
 
 def _add_runs(wandb_runs: list[wapi.Run]):
     for wandb_run in wandb_runs:
-        if not _is_valid_run(wandb_run):
+        version = wandb_run.tags[1][8:]
+        if not _is_valid_run(wandb_run, version):
             continue
 
         metrics = wandb_run.summary
 
-        submission_info: dict[Uid, SubmissionInfo] = {}
-        submissions: dict[Uid, Submission] = {}
-        invalid_submissions: dict[Uid, InvalidSubmission] = {}
+        submission_info: dict[Hotkey, SubmissionInfo] = {}
+        submissions: dict[Hotkey, Submission] = {}
+        invalid_submissions: set[Hotkey] = set()
 
         baseline_metrics: MetricData | None = None
         if "baseline" in metrics:
@@ -152,67 +142,65 @@ def _add_runs(wandb_runs: list[wapi.Run]):
             )
 
         if "submissions" in metrics:
-            for uid, submission in metrics["submissions"].items():
-                submission_info[uid] = SubmissionInfo(
-                    uid=int(uid),
-                    hotkey=submission["hotkey"],
-                    repository=submission["repository"],
-                    revision=submission["revision"],
+            for hotkey, submission in metrics["submissions"].items():
+                uid = UIDS_BY_HOTKEY.get(hotkey)
+                if not uid:
+                    continue
+                submission_info[hotkey] = SubmissionInfo(
+                    uid=uid,
+                    repository=submission["repository_info"]["url"],
+                    revision=submission["repository_info"]["revision"],
                     block=submission["block"],
                 )
 
-        if "benchmarks" in metrics:
-            for uid, benchmark in metrics["benchmarks"].items():
-                model = benchmark["model"]
-                if uid not in submission_info:
+        if "benchmarks" in metrics and "ranks" in metrics:
+            for hotkey, benchmark in metrics["benchmarks"].items():
+                benchmark_metrics = benchmark["metrics"]
+                if hotkey not in submission_info:
                     continue
-                submissions[int(uid)] = Submission(
-                    info=submission_info[uid],
+                submissions[hotkey] = Submission(
+                    info=submission_info[hotkey],
                     metrics=MetricData(
-                        generation_time=float(model["generation_time"]),
-                        vram_used=float(model["vram_used"]),
-                        watts_used=float(model["watts_used"]),
-                        load_time=float(model["load_time"]),
-                        size=int(model["size"]),
+                        generation_time=float(benchmark_metrics["generation_time"]),
+                        vram_used=float(benchmark_metrics["vram_used"]),
+                        watts_used=float(benchmark_metrics["watts_used"]),
+                        load_time=float(benchmark_metrics["load_time"]),
+                        size=int(benchmark_metrics["size"]),
                     ),
                     average_similarity=float(benchmark["average_similarity"]),
                     min_similarity=float(benchmark["min_similarity"]),
-                    tier=int(benchmark["tier"]),
-                    score=float(benchmark["score"]),
+                    rank=int(metrics["ranks"][hotkey]),
+                    score=float(metrics["scores"][hotkey]),
                 )
 
-        if "invalid" in metrics:
-            for uid, reason in metrics["invalid"].items():
-                if not uid in submission_info:
-                    continue
-                invalid_submissions[int(uid)] = InvalidSubmission(
-                    info=submission_info[uid],
-                    reason=reason,
-                )
+        if "invalid_submissions" in metrics:
+            try:
+                for hotkey in metrics["invalid_submissions"]:
+                    invalid_submissions.add(hotkey)
+            except KeyError:
+                ...
 
         status = _status_from_run(wandb_run)
         winners = sorted(
             submissions.values(),
-            key=lambda submission: (submission.tier, -submission.info.block),
-            reverse=True,
+            key=lambda submission: (submission.rank, submission.info.block),
         )
-        winner_uid = winners[0].info.uid if winners and status == status.FINISHED else None
+        winner_uid = winners[0].info.uid if winners else None
 
-        from chain_data import VALIDATOR_IDENTITIES
         uid = int(wandb_run.config["uid"])
         hotkey = wandb_run.config["hotkey"]
         date = _date_from_run(wandb_run)
         id = wandb_run.id
-        average_benchmark_time = float(wandb_run.summary["average_benchmark_time"]) if "average_benchmark_time" in wandb_run.summary else 0
+        average_benchmarking_time = float(wandb_run.summary["average_benchmarking_time"]) if "average_benchmarking_time" in wandb_run.summary else 0
         run = Run(
             start_date=date,
-            version=wandb_run.tags[1][8:],
+            version=version,
             uid=uid,
             name=VALIDATOR_IDENTITIES.get(hotkey, f"{hotkey[:6]}..."),
             hotkey=hotkey,
             status=status,
-            average_benchmark_time=average_benchmark_time,
-            eta=max(int(average_benchmark_time * (len(submission_info) - len(submissions) - len(invalid_submissions))) if average_benchmark_time else 0, 0) if status != BenchmarkStatus.FINISHED else 0,
+            average_benchmarking_time=average_benchmarking_time,
+            eta=max(int(average_benchmarking_time * (len(submission_info) - len(submissions) - len(invalid_submissions))) if average_benchmarking_time else 0, 0) if status != BenchmarkStatus.FINISHED else 0,
             winner_uid=winner_uid,
             baseline_metrics=baseline_metrics,
             total_submissions=len(submission_info),
@@ -264,35 +252,24 @@ def _fetch_current_runs(wandb_api: wandb.Api):
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
 
 
-def sync(timeout: int = 10):
+def sync():
     global last_sync
     now = datetime.now(TIMEZONE)
     if now - last_sync < timedelta(seconds=60):
         return
     last_sync = now
 
-    def sync_task():
-        print("Syncing runs...")
-        wandb_api = wandb.Api()
-        if not RUNS:
-            _fetch_history(wandb_api)
-        else:
-            _fetch_current_runs(wandb_api)
-
-    with ThreadPoolExecutor(max_workers=1) as executor:
-        future = executor.submit(sync_task)
-        try:
-            future.result(timeout=timeout)
-        except TimeoutError:
-            print("Timed out while syncing runs")
-        except Exception as e:
-            print(f"Error occurred while syncing runs: {e}")
+    print("Syncing runs...")
+    wandb_api = wandb.Api()
+    if not RUNS:
+        _fetch_history(wandb_api)
+    else:
+        _fetch_current_runs(wandb_api)
 
 
 def get_current_runs() -> list[Run]:
-    sync()
-    from chain_data import sync_metagraph
     sync_metagraph()
+    sync()
 
     today = _get_contest_start()