Sped up get_data_transferred 7x

utils.py

@@ -1,12 +1,13 @@
+import json
 import os
-import tqdm
 import time
-import wandb
-import streamlit as st
-import pandas as pd
-import bittensor as bt
-import ast
 
+import bittensor as bt
+import numpy as np
+import pandas as pd
+import streamlit as st
+import tqdm
+import wandb
 
 # TODO: Store the runs dataframe (as in sn1 dashboard) and top up with the ones created since the last snapshot
 # TODO: Store relevant wandb data in a database for faster access
@@ -142,11 +143,18 @@ def get_total_md_input_sizes(run):
     return convert_unit(size_bytes, from_unit='B', to_unit=BASE_UNITS)
 
 
+
+
 def get_data_transferred(df, unit='GB'):
+    def safe_json_loads(x):
+        try:
+            return json.loads(x)
+        except ValueError:
+            return []
+
+    validator_sent = np.nansum(df.md_inputs_sizes.dropna().apply(safe_json_loads).explode().replace([np.inf, -np.inf], np.nan).values)
+    miner_sent = np.nansum(df.response_returned_files_sizes.dropna().apply(safe_json_loads).explode().explode().replace([np.inf, -np.inf], np.nan).values)
     
-    validator_sent = df.md_inputs_sizes.dropna().apply(lambda x: ast.literal_eval(x)).explode().sum()
-    miner_sent = df.response_returned_files_sizes.dropna().apply(lambda x: ast.literal_eval(x)).explode().explode().sum() 
- 
     return {
         'validator_sent': convert_unit(validator_sent, from_unit='B', to_unit=BASE_UNITS),
         'miner_sent': convert_unit(miner_sent, from_unit='B', to_unit=BASE_UNITS),