Switch to fiber

requirements.txt

@@ -1,5 +1,5 @@
-bittensor==8.2.1
-gradio==5.1.0
+fiber @ git+https://github.com/rayonlabs/fiber.git@1.0.0#egg=fiber[chain]
+gradio==5.5.0
 wandb==0.18.5
 substrate-interface==1.7.10
 plotly==5.24.1

src/chain_data.py

@@ -1,69 +1,100 @@
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta
+from math import ceil
 from typing import TypeAlias
 
-import bittensor as bt
+from fiber.chain.interface import get_substrate
+from fiber.chain.metagraph import Metagraph
+from fiber.chain.models import Node
+from substrateinterface.storage import StorageKey
 
-from wandb_data import Uid, TIMEZONE
+from wandb_data import Hotkey, Uid, TIMEZONE
 
 Weight: TypeAlias = float
 Incentive: TypeAlias = float
 
 NET_UID = 39
-WEIGHTS_BY_MINER: list[list[tuple[Uid, Weight]]] = []
-INCENTIVES: dict[Uid, Incentive] = {}
-VALIDATOR_IDENTITIES: dict[Uid, str] = {}
-VTRUST: dict[Uid, float] = {}
-UPDATED: dict[Uid, int] = {}
+WEIGHTS_BY_MINER: dict[Hotkey, list[tuple[Hotkey, Weight]]] = {}
+VALIDATOR_IDENTITIES: dict[Hotkey, str] = {}
+UPDATED: dict[Hotkey, int] = {}
 
-subtensor = bt.subtensor()
-metagraph = bt.metagraph(netuid=NET_UID)
+UIDS_BY_HOTKEY: dict[Hotkey, Uid] = {}
+HOTKEYS_BY_UID: dict[Uid, Hotkey] = {}
 
+substrate = get_substrate()
+metagraph = Metagraph(substrate, netuid=NET_UID, load_old_nodes=False)
 
-def fetch_weights():
-    WEIGHTS_BY_MINER.clear()
-    weights = subtensor.weights(netuid=NET_UID)
 
-    for miner_uid in range(metagraph.n.item()):
-        WEIGHTS_BY_MINER.append([])
-        for validator_uid, validator_weights in weights:
-            if not metagraph.validator_permit[validator_uid]:
-                continue
+def query_subtensor(storage_keys: list[StorageKey], block: int) -> list:
+    global substrate
+    try:
+        return substrate.query_multi(
+            storage_keys=storage_keys,
+            block_hash=substrate.get_block_hash(block),
+        )
+    except Exception:
+        substrate = get_substrate()
+        raise
 
-            weight = 0.0
-            for miner_weight in validator_weights:
-                if miner_weight[0] == miner_uid:
-                    weight = miner_weight[1] / 2 ** 16
-                    break
-            WEIGHTS_BY_MINER[miner_uid].append((validator_uid, weight))
+
+def is_validator(node: Node) -> bool:
+    return node.vtrust > 0 or node.stake > 10_000
 
 
-def fetch_incentives():
-    INCENTIVES.clear()
-    for i in range(len(metagraph.incentive)):
-        INCENTIVES[i] = metagraph.incentive[i]
+def get_nodes() -> dict[Hotkey, Node]:
+    return metagraph.nodes
 
 
-def fetch_vtrust():
-    VTRUST.clear()
-    for i in range(len(metagraph.validator_trust)):
-        VTRUST[i] = metagraph.validator_trust[i]
+def fetch_weights(block: int):
+    WEIGHTS_BY_MINER.clear()
+    storage_keys: list[StorageKey] = []
+    for hotkey, node in metagraph.nodes.items():
+        if not is_validator(node): continue
+        storage_keys.append(substrate.create_storage_key(
+            "SubtensorModule",
+            "Weights",
+            [metagraph.netuid, UIDS_BY_HOTKEY[node.hotkey]]
+        ))
+
+    weights = query_subtensor(storage_keys, block)
+
+    for hotkey, node in metagraph.nodes.items():
+        for storage, validator_weights in weights:
+            validator_hotkey = HOTKEYS_BY_UID[storage.params[1]]
+            if hotkey not in WEIGHTS_BY_MINER:
+                WEIGHTS_BY_MINER[hotkey] = []
+            weight = 0.0
+            for miner_weight in validator_weights:
+                if miner_weight[0].value == UIDS_BY_HOTKEY[hotkey]:
+                    weight = miner_weight[1].value / 2 ** 16
+                    break
+            WEIGHTS_BY_MINER[hotkey].append((validator_hotkey, weight))
 
 
 def fetch_updated(block: int):
     UPDATED.clear()
-    for i in range(len(metagraph.last_update)):
-        UPDATED[i] = block - metagraph.last_update[i]
+    for hotkey, node in metagraph.nodes.items():
+        UPDATED[hotkey] = ceil(block - node.last_updated)
 
 
-def fetch_identities():
+def fetch_identities(block: int):
     VALIDATOR_IDENTITIES.clear()
-    for uid in range(metagraph.n.item()):
-        if not metagraph.validator_permit[uid]:
-            continue
-        identity = subtensor.substrate.query('SubtensorModule', 'Identities', [metagraph.coldkeys[uid]])
-        if identity != None:
-            VALIDATOR_IDENTITIES[uid] = identity.value["name"]
+    storage_keys: list[StorageKey] = []
+    for hotkey, node in metagraph.nodes.items():
+        if not is_validator(node): continue
+        storage_keys.append(substrate.create_storage_key(
+            "SubtensorModule",
+            "Identities",
+            [node.coldkey]
+        ))
+
+    identities = query_subtensor(storage_keys, block)
+    for hotkey, node in metagraph.nodes.items():
+        for storage, info in identities:
+            if node.coldkey != storage.params[0]: continue
+            if info != None: # noqa
+                VALIDATOR_IDENTITIES[hotkey] = info.value["name"]
+                break
 
 
 last_sync: datetime = datetime.fromtimestamp(0, TIMEZONE)
@@ -79,12 +110,14 @@ def sync_metagraph(timeout: int = 10):
 
     def sync_task():
         print("Syncing metagraph...")
-        block = subtensor.get_current_block()
-        metagraph.sync(block=block, subtensor=subtensor)
+        block = substrate.get_block_number(None)  # type: ignore
+        metagraph.sync_nodes()
+
+        for uid, node in enumerate(metagraph.nodes.values()):
+            UIDS_BY_HOTKEY[node.hotkey] = uid
+            HOTKEYS_BY_UID[uid] = node.hotkey
 
-        fetch_weights()
-        fetch_incentives()
-        fetch_vtrust()
+        fetch_weights(block)
         fetch_updated(block)
 
         global last_identity_sync
@@ -92,7 +125,7 @@ def sync_metagraph(timeout: int = 10):
             return
 
         last_identity_sync = now
-        fetch_identities()
+        fetch_identities(block)
 
     with ThreadPoolExecutor(max_workers=1) as executor:
         future = executor.submit(sync_task)

src/validator_states.py

@@ -6,7 +6,7 @@ import pandas as pd
 from packaging import version
 
 from wandb_data import get_current_runs, Run
-from chain_data import VTRUST, UPDATED
+from chain_data import get_nodes, UPDATED
 
 AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 180  # 3 minutes
 ETA_WARNING_THRESHOLD = 43200  # 12 hours
@@ -34,8 +34,8 @@ def create_validator_states() -> gr.Dataframe:
     winner_uid_mode = statistics.mode(winners) if winners else None
     latest_version = get_latest_version(runs)
     for run in runs:
-        vtrust = VTRUST.get(run.uid, 0)
-        updated = UPDATED.get(run.uid, 0)
+        vtrust = get_nodes().get(run.hotkey, 0).vtrust
+        updated = UPDATED.get(run.hotkey, 0)
         data.append([
             run.uid,
             run.name,

src/validator_weights.py

@@ -1,8 +1,8 @@
 import gradio as gr
 import pandas as pd
 
-from chain_data import WEIGHTS_BY_MINER, INCENTIVES, sync_metagraph, Weight
-from wandb_data import Uid, get_current_runs
+from chain_data import WEIGHTS_BY_MINER, UIDS_BY_HOTKEY, get_nodes, sync_metagraph, Weight
+from wandb_data import Hotkey, get_current_runs
 
 
 def get_color_by_weight(weight: float) -> str:
@@ -22,16 +22,16 @@ def get_color_by_weight(weight: float) -> str:
         g = int(255 - ((1 - progress) * 50))
         return f"rgb(0, {g}, 0)"
 
-def get_active_weights() -> list[list[tuple[Uid, Weight]]]:
+def get_active_weights() -> dict[Hotkey, list[tuple[Hotkey, Weight]]]:
     runs = get_current_runs()
-    weights: list[list[tuple[Uid, Weight]]] = []
+    weights: dict[Hotkey, list[tuple[Hotkey, Weight]]] = {}
 
-    for miner_uid, validator_weights in enumerate(WEIGHTS_BY_MINER):
-        new_weights: list[tuple[Uid, Weight]] = []
-        for validator_uid, weight in validator_weights:
-            if validator_uid in [run.uid for run in runs]:
-                new_weights.append((validator_uid, weight))
-        weights.append(new_weights)
+    for hotkey, validator_weights in WEIGHTS_BY_MINER.items():
+        new_weights: list[tuple[Hotkey, Weight]] = []
+        for validator_hotkey, weight in validator_weights:
+            if validator_hotkey in [run.hotkey for run in runs]:
+                new_weights.append((validator_hotkey, weight))
+        weights[hotkey] = new_weights
 
     return weights
 
@@ -45,19 +45,17 @@ def create_weights(include_inactive: bool) -> gr.Dataframe:
     weights = WEIGHTS_BY_MINER if include_inactive else get_active_weights()
 
     validator_uids = set()
-    for validator_weights in weights:
-        for validator_uid, _ in validator_weights:
-            validator_uids.add(validator_uid)
+    for _, validator_weights in weights.items():
+        for validator_hotkey, _ in validator_weights:
+            validator_uids.add(UIDS_BY_HOTKEY[validator_hotkey])
 
     for validator_uid in sorted(validator_uids):
         headers.append(str(validator_uid))
         datatype.append("markdown")
 
-    for miner_uid, validator_weights in enumerate(weights):
-        if miner_uid not in INCENTIVES:
-            break
-        incentive = INCENTIVES[miner_uid]
-        row = [miner_uid, f"<span style='color: {get_color_by_weight(incentive)}'>{incentive:.{3}f}</span>"]
+    for hotkey, validator_weights in weights.items():
+        incentive = get_nodes().get(hotkey, 0).incentive / 2 ** 16
+        row = [UIDS_BY_HOTKEY[hotkey], f"<span style='color: {get_color_by_weight(incentive)}'>{incentive:.{3}f}</span>"]
         for _, weight in validator_weights:
             row.append(f"<span style='color: {get_color_by_weight(weight)}'>{weight:.{3}f}</span>")
         data.append(row)

src/wandb_data.py

@@ -16,6 +16,7 @@ TIMEZONE = ZoneInfo("America/Los_Angeles")
 START_DATE = datetime(2024, 11, 9)
 
 Uid: TypeAlias = int
+Hotkey: TypeAlias = str
 
 
 class BenchmarkStatus(Enum):
@@ -50,7 +51,7 @@ class MetricData:
 @dataclass
 class SubmissionInfo:
     uid: int
-    hotkey: str
+    hotkey: Hotkey
     repository: str
     revision: str
     block: int
@@ -207,7 +208,7 @@ def _add_runs(wandb_runs: list[wapi.Run]):
             start_date=date,
             version=wandb_run.tags[1][8:],
             uid=uid,
-            name=VALIDATOR_IDENTITIES.get(uid, f"{hotkey[:6]}..."),
+            name=VALIDATOR_IDENTITIES.get(hotkey, f"{hotkey[:6]}..."),
             hotkey=hotkey,
             status=status,
             average_benchmark_time=average_benchmark_time,