api-dashboard

api.py

@@ -2,37 +2,26 @@
 import atexit
 import datetime
 
-import pandas as pd
-import uvicorn
+from flask import Flask, request, jsonify
 from apscheduler.schedulers.background import BackgroundScheduler
-from fastapi import FastAPI
 
 import utils
-from classes import Metagraph, Productivity, Throughput
+
+app = Flask(__name__)
+
 
 # Global variables (saves time on loading data)
 state_vars = None
 reload_timestamp = datetime.datetime.now().strftime('%D %T')
 
-data_all = None
-data_30d = None
-data_24h = None
-
-app = FastAPI()
 
 def load_data():
     """
     Reload the state variables
     """
-    global data_all, data_30d ,data_24h, reload_timestamp
-    
-    utils.fetch_new_runs()
-    
-    data_all = utils.preload_data()
-    
-    data_30d = data_all[(pd.Timestamp.now() -  data_all['updated_at'].apply(lambda x: pd.Timestamp(x)) < pd.Timedelta('30 days'))]
-    
-    data_24h = data_all[(pd.Timestamp.now() -  data_all['updated_at'].apply(lambda x: pd.Timestamp(x)) < pd.Timedelta('1 days'))]
+    global state_vars, reload_timestamp
+    state_vars = utils.load_state_vars()
+
     reload_timestamp = datetime.datetime.now().strftime('%D %T')
 
     print(f'Reloaded data at {reload_timestamp}')
@@ -47,65 +36,110 @@ def start_scheduler():
     atexit.register(lambda: scheduler.shutdown())
 
 
-@app.get("/")
+@app.route('/', methods=['GET'])
 def home():
     return "Welcome to the Bittensor Protein Folding Leaderboard API!"
 
 
-@app.get("/updated")
+@app.route('/updated', methods=['GET'])
 def updated():
     return reload_timestamp
 
 
-@app.get("/productivity", response_model=Productivity)
-def productivity_metrics():
+@app.route('/data', methods=['GET'])
+@app.route('/data/<period>', methods=['GET'])
+def data(period=None):
+    """
+    Get the productivity metrics
+    """
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(
+        df.astype(str).to_dict(orient='records')
+    )
+
+@app.route('/productivity', methods=['GET'])
+@app.route('/productivity/<period>', methods=['GET'])
+def productivity_metrics(period=None):
     """
     Get the productivity metrics
     """
 
-    result = utils.get_productivity(df_all=data_all, df_24h=data_24h, df_30d=data_30d)
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(
+        utils.get_productivity(df)
+    )
 
 
-    return result
-@app.get("/metagraph", response_model=Metagraph)
-def get_metagraph():
+@app.route('/throughput', methods=['GET'])
+@app.route('/throughput/<period>', methods=['GET'])
+def throughput_metrics(period=None):
     """
-    Get the metagraph
+    Get the throughput metrics
     """
-    
-    df_m = utils.get_metagraph()
-    df_miners = df_m.sort_values('I', ascending=False).reset_index()
-    incentives = df_miners['I'].astype(float).values
-    emissions = df_miners['E'].astype(float).values
-    identities = df_miners['identity']
-    hotkeys = df_miners['hotkey']
-    coldkeys = df_miners['coldkey']
-    trusts = df_miners['trust'].astype(float).values
-    results = {'incentives': incentives,
-               'emissions': emissions,
-               'identities': identities,
-               'hotkeys': hotkeys,
-               'coldkeys': coldkeys,
-               'trusts': trusts}
-    return results
-
-@app.get("/throughput", response_model=Throughput)
-def throughput_metrics():
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(utils.get_data_transferred(df))
+
+
+@app.route('/metagraph', methods=['GET'])
+def metagraph():
     """
-    Get the throughput metrics
+    Get the metagraph data
+    Returns:
+    - metagraph_data: List of dicts (from pandas DataFrame)
+    """
+
+    df_m = state_vars["metagraph"]
+
+    return jsonify(
+        df_m.to_dict(orient='records')
+    )
+
+@app.route('/leaderboard', methods=['GET'])
+@app.route('/leaderboard/<entity>', methods=['GET'])
+@app.route('/leaderboard/<entity>/<ntop>', methods=['GET'])
+def leaderboard(entity='identity',ntop=10):
+    """
+    Get the leaderboard data
+    Returns:
+    - leaderboard_data: List of dicts (from pandas DataFrame)
     """
-    
-    return utils.get_data_transferred(data_all, data_24h)
 
+    assert entity in utils.ENTITY_CHOICES, f"Invalid entity choice: {entity}"
+
+    df_miners = utils.get_leaderboard(
+        state_vars["metagraph"],
+        ntop=int(ntop),
+        entity_choice=entity
+        )
+
+    return jsonify(
+        df_miners.to_dict(orient='records')
+    )
+
+@app.route('/validator', methods=['GET'])
+def validator():
+    """
+    Get the validator data
+    Returns:
+    - validator_data: List of dicts (from pandas DataFrame)
+    """
+    df_m = state_vars["metagraph"]
+    df_validators = df_m.loc[df_m.validator_trust > 0]
+
+    return jsonify(
+        df_validators.to_dict(orient='records')
+    )
 
 
 if __name__ == '__main__':
 
     load_data()
-
     start_scheduler()
 
-    uvicorn.run(app, host='0.0.0.0', port=5001)
+    app.run(host='0.0.0.0', port=5001, debug=True)
     
 
     # to test locally

app.py

@@ -1,9 +1,7 @@
 import time
-
 import pandas as pd
-import plotly.express as px
-import requests
 import streamlit as st
+import plotly.express as px
 
 import utils
 
@@ -16,26 +14,22 @@ Simulation duration distribution
 """
 
 UPDATE_INTERVAL = 3600
-BASE_URL = 'http://143.198.21.86:5001/'
+
 
 st.title('Folding Subnet Dashboard')
 st.markdown('<br>', unsafe_allow_html=True)
 
-@st.cache_data(ttl=UPDATE_INTERVAL)
-def fetch_productivity_data():
-    return requests.get(f'{BASE_URL}/productivity').json()
-
-@st.cache_data(ttl=UPDATE_INTERVAL)
-def fetch_throughput_data():
-    return requests.get(f'{BASE_URL}/throughput').json()
+# reload data periodically
+df = utils.build_data(time.time()//UPDATE_INTERVAL)
+st.toast(f'Loaded {len(df)} runs')
 
-@st.cache_data(ttl=UPDATE_INTERVAL)
-def fetch_metagraph_data():
-    return utils.get_metagraph()
+# TODO: fix the factor for 24 hours ago
+runs_alive_24h_ago = (df.last_event_at > pd.Timestamp.now() - pd.Timedelta('1d'))
+df_24h = df.loc[runs_alive_24h_ago]
+# correction factor to account for the fact that the data straddles the 24h boundary
+# correction factor is based on the fraction of the run which occurred in the last 24h
+# factor = (df_24h.last_event_at - pd.Timestamp.now() + pd.Timedelta('1d')) / pd.Timedelta('1d')
 
-@st.cache_data(ttl=UPDATE_INTERVAL)
-def fetch_leaderboard_data(df_m, ntop, entity_choice):
-    return utils.get_leaderboard(df_m, entity_choice=entity_choice)
 
 #### ------ PRODUCTIVITY ------
 
@@ -43,84 +37,68 @@ def fetch_leaderboard_data(df_m, ntop, entity_choice):
 st.subheader('Productivity overview')
 st.info('Productivity metrics show how many proteins have been folded, which is the primary goal of the subnet. Metrics are estimated using weights and biases data combined with heuristics.')
 
-productivity_all = fetch_productivity_data()
-completed_jobs = productivity_all['all_time']['total_completed_jobs_data']
-
-productivity_24h = productivity_all['last_24h']
-completed_jobs = pd.DataFrame(completed_jobs)
+productivity = utils.get_productivity(df)
+productivity_24h = utils.get_productivity(df_24h)
 
-unique_folded = pd.DataFrame(productivity_all['all_time']['unique_folded_data'])
-# unique_folded['last_event_at'] = pd.to_datetime(unique_folded['updated_at'])
 
 m1, m2, m3 = st.columns(3)
-m1.metric('Unique proteins folded', f'{len(unique_folded):,.0f}', delta=f'{productivity_24h["unique_folded"]:,.0f} (24h)')
-m2.metric('Total jobs completed', f'{len(completed_jobs):,.0f}', delta=f'{productivity_24h["total_completed_jobs"]:,.0f} (24h)')
-m3.metric('Total simulations ran', f'{len(completed_jobs)*10:,.0f}', delta=f'{productivity_24h["total_completed_jobs"]*10:,.0f} (24h)')
+m1.metric('Unique proteins folded', f'{productivity.get("unique_folded"):,.0f}', delta=f'{productivity_24h.get("unique_folded"):,.0f} (24h)')
+m2.metric('Total proteins folded', f'{productivity.get("total_simulations"):,.0f}', delta=f'{productivity_24h.get("total_simulations"):,.0f} (24h)')
+m3.metric('Total simulation steps', f'{productivity.get("total_md_steps"):,.0f}', delta=f'{productivity_24h.get("total_md_steps"):,.0f} (24h)')
+
 st.markdown('<br>', unsafe_allow_html=True)
 
+time_binned_data = df.set_index('last_event_at').groupby(pd.Grouper(freq='12h'))
+
 PROD_CHOICES = {
-    'Total jobs completed': 'total_pdbs',
     'Unique proteins folded': 'unique_pdbs',
+    'Total simulations': 'total_pdbs',
+    'Total simulation steps': 'total_md_steps',
 }
-
 prod_choice_label = st.radio('Select productivity metric', list(PROD_CHOICES.keys()), index=0, horizontal=True)
 prod_choice = PROD_CHOICES[prod_choice_label]
-
-PROD_DATA = {
-    'unique_pdbs': unique_folded,
-    'total_pdbs': completed_jobs,
-}
-df = PROD_DATA[prod_choice]
-
-df = df.sort_values(by='last_event_at').reset_index()
-
-# Create a cumulative count column
-df['cumulative_jobs'] = df.index + 1
-
-# Plot the cumulative jobs over time
+steps_running_total = time_binned_data[prod_choice].sum().cumsum()
 st.plotly_chart(
-    px.line(df, x='last_event_at', y='cumulative_jobs', 
-              labels={'last_event_at': 'Time', 'cumulative_jobs': prod_choice_label}).update_traces(fill='tozeroy'),
+    # add fillgradient to make it easier to see the trend
+    px.area(steps_running_total, y=prod_choice, 
+            labels={'last_event_at':'', prod_choice: prod_choice_label},
+    ).update_traces(fill='tozeroy'),
     use_container_width=True,
 )
 
 st.markdown('<br>', unsafe_allow_html=True)
 
+
 #### ------ THROUGHPUT ------
 st.subheader('Throughput overview')
 
 st.info('Throughput metrics show the total amount of data sent and received by the validators. This is a measure of the network activity and the amount of data that is being processed by the subnet.')
 
 MEM_UNIT = 'GB' #st.radio('Select memory unit', ['TB','GB', 'MB'], index=0, horizontal=True)
-throughput = fetch_throughput_data()
 
-data_transferred = throughput['all_time']
-data_transferred_24h = throughput['last_24h']
-data_df = pd.DataFrame(throughput['data'])
-data_df = data_df.sort_values(by='updated_at').reset_index()
-data_df['updated_at'] = pd.to_datetime(data_df['updated_at'])
-data_df['Total validator data sent'] = data_df['md_inputs_sum'].cumsum()
-data_df['Total received data'] = data_df['md_outputs_sum'].cumsum()
+data_transferred = utils.get_data_transferred(df,unit=MEM_UNIT)
+data_transferred_24h = utils.get_data_transferred(df_24h, unit=MEM_UNIT)
 
 m1, m2, m3 = st.columns(3)
-m1.metric(f'Total validator data sent ({MEM_UNIT})', f'{data_transferred["validator_sent"]:,.0f}', delta=f'{data_transferred_24h["validator_sent"]:,.0f} (24h)')
-m2.metric(f'Total received data ({MEM_UNIT})', f'{data_transferred["miner_sent"]:,.0f}', delta=f'{data_transferred_24h["miner_sent"]:,.0f} (24h)')
-m3.metric(f'Total transferred data ({MEM_UNIT})', f'{data_transferred["validator_sent"]+data_transferred["miner_sent"]:,.0f}', delta=f'{data_transferred_24h["validator_sent"]+data_transferred_24h["miner_sent"]:,.0f} (24h)')
+m1.metric(f'Total sent data ({MEM_UNIT})', f'{data_transferred.get("sent"):,.0f}', delta=f'{data_transferred_24h.get("sent"):,.0f} (24h)')
+m2.metric(f'Total received data ({MEM_UNIT})', f'{data_transferred.get("received"):,.0f}', delta=f'{data_transferred_24h.get("received"):,.0f} (24h)')
+m3.metric(f'Total transferred data ({MEM_UNIT})', f'{data_transferred.get("total"):,.0f}', delta=f'{data_transferred_24h.get("total"):,.0f} (24h)')
+
+
+IO_CHOICES = {'total_data_sent':'Sent', 'total_data_received':'Received'}
+io_running_total = time_binned_data[list(IO_CHOICES.keys())].sum().rename(columns=IO_CHOICES).cumsum().melt(ignore_index=False)
+io_running_total['value'] = io_running_total['value'].apply(utils.convert_unit, args=(utils.BASE_UNITS, MEM_UNIT))
 
 st.plotly_chart(
-    px.line(data_df, x='updated_at', y=['Total validator data sent', 'Total received data'],
-            labels={'updated_at':'Time', 'value':f'Data Transferred ({MEM_UNIT})', 'variable':'Direction'},
-            ).update_traces(fill='tozeroy').update_layout(legend=dict(
-    yanchor="top",
-    y=0.99,
-    xanchor="left",
-    x=0.01
-)),
+    px.area(io_running_total, y='value', color='variable',
+            labels={'last_event_at':'', 'value': f'Data transferred ({MEM_UNIT})', 'variable':'Direction'},
+    ),
     use_container_width=True,
 )
 
 st.markdown('<br>', unsafe_allow_html=True)
 
+
 #### ------ LEADERBOARD ------
 
 st.subheader('Leaderboard')
@@ -129,31 +107,34 @@ m1, m2 = st.columns(2)
 ntop = m1.slider('Number of top miners to display', value=10, min_value=3, max_value=50, step=1)
 entity_choice = m2.radio('Select entity', utils.ENTITY_CHOICES, index=0, horizontal=True)
 
-df_m = fetch_metagraph_data()
-df_miners = fetch_leaderboard_data(df_m, ntop=ntop, entity_choice=entity_choice)
+df_m = utils.get_metagraph(time.time()//UPDATE_INTERVAL)
+df_miners = utils.get_leaderboard(df_m, ntop=ntop, entity_choice=entity_choice)
+
 # hide colorbar and don't show y axis
 st.plotly_chart(
-    px.bar(df_miners.iloc[-ntop:], x='I', color='I', hover_name=entity_choice, text=entity_choice if ntop < 20 else None,
+    px.bar(df_miners, x='I', color='I', hover_name=entity_choice, text=entity_choice if ntop < 20 else None,
             labels={'I':'Incentive', 'trust':'Trust', 'stake':'Stake', '_index':'Rank'},
     ).update_layout(coloraxis_showscale=False, yaxis_visible=False),
     use_container_width=True,
 )
 
+
 with st.expander('Show raw metagraph data'):
     st.dataframe(df_m)
 
 st.markdown('<br>', unsafe_allow_html=True)
 
+
 #### ------ LOGGED RUNS ------
 
-# st.subheader('Logged runs')
-# st.info('The timeline shows the creation and last event time of each run.')
-# st.plotly_chart(
-#     px.timeline(df, x_start='created_at', x_end='last_event_at', y='username', color='state',
-#                 labels={'created_at':'Created at', 'last_event_at':'Last event at', 'username':''},
-#                 ),
-#     use_container_width=True
-# )
-
-# with st.expander('Show raw run data'):
-#     st.dataframe(df)
+st.subheader('Logged runs')
+st.info('The timeline shows the creation and last event time of each run.')
+st.plotly_chart(
+    px.timeline(df, x_start='created_at', x_end='last_event_at', y='username', color='state',
+                labels={'created_at':'Created at', 'last_event_at':'Last event at', 'username':''},
+                ),
+    use_container_width=True
+)
+
+with st.expander('Show raw run data'):
+    st.dataframe(df)

classes.py

@@ -1,37 +0,0 @@
-from pydantic import BaseModel
-from datetime import datetime
-from typing import List
-
-
-class Data(BaseModel):
-    last_event_at: List[datetime]
-    cumulative_jobs: List[int]
-
-class ProductivityData(BaseModel):
-    unique_folded: int
-    total_completed_jobs: int
-    unique_folded_data: Data
-    total_completed_jobs_data: Data
-    
-
-class Productivity(BaseModel):
-    all_time: ProductivityData
-    last_24h: ProductivityData
-    last_30d: ProductivityData
-    
-class ThroughputData(BaseModel):
-    validator_sent: float
-    miner_sent: float
-
-class Throughput(BaseModel):
-    all_time: ThroughputData
-    last_24h: ThroughputData
-    data: dict
-    
-class Metagraph(BaseModel):
-    incentives: List[float]
-    emissions: List[float]
-    identities: List[str]
-    hotkeys: List[str]
-    coldkeys: List[str]
-    trusts: List[float]

ecosystem.config.js

@@ -1,14 +0,0 @@
-module.exports = {
-  apps: [
-    {
-      name: 'hf-dashboard-api',   
-      script: '/home/spunion/Sn25/api.py',   
-      interpreter: '/home/spunion/Sn25/venv/bin/python', 
-      autorestart: true,    
-      watch: false,         
-      env: {
-        NODE_ENV: 'production',
-      },
-    },
-  ],
-};

requirements.txt

@@ -7,7 +7,5 @@ streamlit
 nbformat
 plotly
 pandas
-pydantic
-fastapi
-uvicorn
+flask
 

utils.py

@@ -1,13 +1,12 @@
-import json
 import os
-import time
-
-import bittensor as bt
-import numpy as np
-import pandas as pd
-import streamlit as st
 import tqdm
+import time
 import wandb
+import streamlit as st
+import pandas as pd
+import bittensor as bt
+
+
 # TODO: Store the runs dataframe (as in sn1 dashboard) and top up with the ones created since the last snapshot
 # TODO: Store relevant wandb data in a database for faster access
 
@@ -16,7 +15,7 @@ import wandb
 MIN_STEPS = 12 # minimum number of steps in wandb run in order to be worth analyzing
 MAX_RUNS = 100#0000
 NETUID = 25
-BASE_PATH = 'macrocosmos/folding-validators' # added historical data from otf wandb and current data
+BASE_PATHS = ['macrocosmos/folding-validators--moved', 'macrocosmos/folding-validators'] # added historical data from otf wandb and current data
 NETWORK = 'finney'
 KEYS = None
 ABBREV_CHARS = 8
@@ -24,12 +23,7 @@ ENTITY_CHOICES = ('identity', 'hotkey', 'coldkey')
 
 PDBS_PER_RUN_STEP = 0.083
 AVG_MD_STEPS = 30_000
-BASE_UNITS = 'GB'
-SAVE_PATH = 'current_runs/'
-# Check if the directory exists
-if not os.path.exists(SAVE_PATH):
-    # If it doesn't exist, create the directory
-    os.makedirs(SAVE_PATH)
+BASE_UNITS = 'MB'
 
 api = wandb.Api(timeout=120, api_key='cdcbe340bb7937d3a289d39632491d12b39231b7')
 
@@ -53,24 +47,24 @@ EXTRACTORS = {
     'run_id': lambda x: x.id,
     'user': lambda x: x.user.name[:16],
     'username': lambda x: x.user.username[:16],
-    # 'created_at': lambda x: pd.Timestamp(x.created_at),
-    'last_event_at': lambda x: pd.to_datetime(x.summary.get('_timestamp'), errors='coerce'),
+    'created_at': lambda x: pd.Timestamp(x.created_at),
+    'last_event_at': lambda x: pd.Timestamp(x.summary.get('_timestamp'), unit='s'),
 
     'netuid': lambda x: x.config.get('netuid'),
     'mock': lambda x: x.config.get('neuron').get('mock'),
     'sample_size': lambda x: x.config.get('neuron').get('sample_size'),
     'queue_size': lambda x: x.config.get('neuron').get('queue_size'),
     'timeout': lambda x: x.config.get('neuron').get('timeout'),
-    # 'update_interval': lambda x: x.config.get('neuron').get('update_interval'),
+    'update_interval': lambda x: x.config.get('neuron').get('update_interval'),
     'epoch_length': lambda x: x.config.get('neuron').get('epoch_length'),
     'disable_set_weights': lambda x: x.config.get('neuron').get('disable_set_weights'),
 
     # This stuff is from the last logged event
     'num_steps': lambda x: x.summary.get('_step'),
-    # 'runtime': lambda x: x.summary.get('_runtime'),
-    # 'init_energy': lambda x: x.summary.get('init_energy'),
-    # 'best_energy': lambda x: x.summary.get('best_loss'),
-    # 'pdb_id': lambda x: x.summary.get('pdb_id'),
+    'runtime': lambda x: x.summary.get('_runtime'),
+    'init_energy': lambda x: x.summary.get('init_energy'),
+    'best_energy': lambda x: x.summary.get('best_loss'),
+    'pdb_id': lambda x: x.summary.get('pdb_id'),
     'pdb_updates': lambda x: x.summary.get('updated_count'),
     'total_returned_sizes': lambda x: get_total_file_sizes(x),
     'total_sent_sizes': lambda x: get_total_md_input_sizes(x),
@@ -80,12 +74,10 @@ EXTRACTORS = {
     'version': lambda x: x.tags[0],
     'spec_version': lambda x: x.tags[1],
     'vali_hotkey': lambda x: x.tags[2],
-    
+
     # System metrics
     'disk_read': lambda x: x.system_metrics.get('system.disk.in'),
     'disk_write': lambda x: x.system_metrics.get('system.disk.out'),
-    'network_sent': lambda x: x.system_metrics.get('system.network.sent'),
-    'network_recv': lambda x: x.system_metrics.get('system.network.recv'),
     # Really slow stuff below
     # 'started_at': lambda x: x.metadata.get('startedAt'),
     # 'disk_used': lambda x: x.metadata.get('disk').get('/').get('used'),
@@ -142,189 +134,44 @@ def get_total_md_input_sizes(run):
     return convert_unit(size_bytes, from_unit='B', to_unit=BASE_UNITS)
 
 
+def get_data_transferred(df, unit='GB'):
 
-
-def get_data_transferred(df, df_24h, unit='GB'):
-    def safe_json_loads(x):
-        try:
-            return json.loads(x)
-        except ValueError:
-            return []
-    def np_sum(x):
-        try:
-            # Flatten the list of lists and convert it to a NumPy array
-            flat_array = np.array([item for sublist in x for item in sublist])
-
-            # Use np.sum() to sum all elements in the flattened array
-            total_sum = np.sum(flat_array)
-            return total_sum
-        except TypeError:
-            return 0
-    df = df.dropna(subset=['md_inputs_sizes', 'response_returned_files_sizes'])
-    df['md_inputs_sizes'] = df.md_inputs_sizes.apply(safe_json_loads)
-    df['response_returned_files_sizes'] = df.response_returned_files_sizes.apply(safe_json_loads)
-    df['md_inputs_sum'] = df.md_inputs_sizes.apply(np.sum)
-    df['md_outputs_sum'] = df.response_returned_files_sizes.apply(np_sum)
-    df['md_inputs_sum'] = df['md_inputs_sum'].apply(convert_unit, from_unit='B', to_unit=BASE_UNITS)
-    df['md_outputs_sum'] = df['md_outputs_sum'].apply(convert_unit, from_unit='B', to_unit=BASE_UNITS)
-    
-    df_24h = df_24h.dropna(subset=['md_inputs_sizes', 'response_returned_files_sizes'])
-    df_24h['md_inputs_sizes'] = df_24h.md_inputs_sizes.apply(safe_json_loads)
-    df_24h['response_returned_files_sizes'] = df_24h.response_returned_files_sizes.apply(safe_json_loads)
-    df_24h['md_inputs_sum'] = df_24h.md_inputs_sizes.apply(np.sum)
-    df_24h['md_outputs_sum'] = df_24h.response_returned_files_sizes.apply(np_sum)
-   
-    
-    validator_sent = np.nansum(df['md_inputs_sum'].values)
-    miner_sent = np.nansum(df['md_outputs_sum'].values)
-    validator_sent_24h = np.nansum(df_24h['md_inputs_sum'].values)
-    miner_sent_24h = np.nansum(df_24h['md_outputs_sum'].values)
-    
-    return {'all_time': {
-        'validator_sent': validator_sent,
-        'miner_sent': miner_sent,
-    },
-    'last_24h': {
-        'validator_sent': convert_unit(validator_sent_24h, from_unit='B', to_unit=BASE_UNITS),
-        'miner_sent': convert_unit(miner_sent_24h, from_unit='B', to_unit=BASE_UNITS),
-    },
-    'data': df[['md_inputs_sum', 'md_outputs_sum', 'updated_at']].to_dict()
-    }
-
-def calculate_productivity_data(df):
-    completed_jobs = df[df['updated_count'] == 10]
-    completed_jobs['last_event_at'] = pd.to_datetime(completed_jobs['updated_at'])
-    unique_folded = completed_jobs.drop_duplicates(subset=['pdb_id'], keep='first')
-    completed_jobs = completed_jobs.sort_values(by='last_event_at').reset_index()
-    completed_jobs['cumulative_jobs'] = completed_jobs.index + 1
-    unique_folded = unique_folded.sort_values(by='last_event_at').reset_index()
-    unique_folded['cumulative_jobs'] = unique_folded.index + 1
+    factor = convert_unit(1, from_unit=BASE_UNITS, to_unit=unit)
+    sent = df.total_data_sent.sum()
+    received = df.total_data_received.sum()
     return {
-        'unique_folded': len(unique_folded),
-        'total_completed_jobs': len(completed_jobs),
-        'unique_folded_data': {'last_event_at': unique_folded['last_event_at'].dt.to_pydatetime(), 'cumulative_jobs':unique_folded['cumulative_jobs'].values},
-        'total_completed_jobs_data': {'last_event_at': completed_jobs['last_event_at'].dt.to_pydatetime(), 'cumulative_jobs':completed_jobs['cumulative_jobs'].values}
-    }
-
-def get_productivity(df_all, df_24h, df_30d): 
-    result = {
-        'all_time': {
-            'unique_folded': 0,
-            'total_completed_jobs': 0,
-            'unique_folded_data': {},
-            'total_completed_jobs_data': {}
-        },
-        'last_24h': {
-            'unique_folded': 0,
-            'total_completed_jobs': 0,
-            "unique_folded_data": {},
-            'total_completed_jobs_data': {}
-        },
-        'last_30d': {
-            'unique_folded': 0,
-            'total_completed_jobs': 0,
-            "unique_folded_data": {},
-            'total_completed_jobs_data': {}
+        'sent':sent * factor,
+        'received':received * factor,
+        'total': (sent + received) * factor,
+        'read':df.disk_read.sum() * factor,
+        'write':df.disk_write.sum() * factor,
         }
-    }
 
 
+def get_productivity(df):
 
-    if df_all is not None:      
-        result['all_time'].update(calculate_productivity_data(df_all))
-        
-    if df_24h is not None:
-        result['last_24h'].update(calculate_productivity_data(df_24h))
-        
-    if df_30d is not None:
-        result['last_30d'].update(calculate_productivity_data(df_30d))
-    return result
+    # Estimate the number of unique pdbs folded using our heuristic
+    unique_folded = df.unique_pdbs.sum().round()
+    # Estimate the total number of simulations completed using our heuristic
+    total_simulations = df.total_pdbs.sum().round()
+    # Estimate the total number of simulation steps completed using our heuristic
+    total_md_steps = df.total_md_steps.sum().round()
 
-def get_leaderboard(df, entity_choice='identity'):
+    return {
+        'unique_folded': unique_folded,
+        'total_simulations': total_simulations,
+        'total_md_steps': total_md_steps,
+    }
+
+def get_leaderboard(df, ntop=10, entity_choice='identity'):
 
     df = df.loc[df.validator_permit==False]
     df.index = range(df.shape[0])
-    return df.groupby(entity_choice).I.sum().sort_values().reset_index()
-
-
-
-def fetch_new_runs(base_path: str = BASE_PATH , netuid: int = 25, min_steps: int = 10, save_path: str= SAVE_PATH, extractors: dict = EXTRACTORS):
-    runs_checker = pd.read_csv('runs_checker.csv')
-    current_time = pd.to_datetime(time.time(), unit='s')
-    current_time_str = current_time.strftime('%y-%m-%d')  # Format as 'YYYYMMDD'
-    new_ticker = runs_checker.check_ticker.max() + 1
-    
-    new_rows_list = []
-
-    # update runs list based on all current runs running    
-    for run in api.runs(base_path):
-        num_steps = run.summary.get('_step')
-
-        if run.config.get('netuid') != netuid:
-            continue
-    
-        if num_steps is None or num_steps < min_steps:
-            continue 
-        
-        if run.state =='running':
-            new_rows_list.append({
-                'run_id': run.id,
-                'state': run.state,
-                'step': num_steps,
-                'check_time': current_time,
-                'check_ticker': new_ticker,
-                'user': run.user.name[:16],
-                'username': run.user.username[:16]
-            })
-    if new_rows_list:
-        new_rows_df = pd.DataFrame(new_rows_list)
-        runs_checker= pd.concat([runs_checker, new_rows_df], ignore_index=True)
-        # save 
-        runs_checker.to_csv('runs_checker.csv', index=False)
-    
-    bt.logging.info(f'Cross checking runs for ticker {new_ticker} against previous ticker')
-    previous_check = runs_checker[runs_checker.check_ticker==new_ticker - 1]
-    current_check = runs_checker[runs_checker.check_ticker == new_ticker]
-
-    # save ended runs from last check  
-    for run_id in previous_check.run_id:
-        if run_id not in current_check.run_id:
-
-            frame = load_run(f'{base_path}/{run_id}', extractors=EXTRACTORS)
-
-            csv_path = os.path.join(save_path, f"{run_id}.csv")
-            frame.to_csv(csv_path)
-
-    # save new runs 
-    for run in api.runs(base_path):
-        if run.config.get('netuid') != netuid:
-            continue
-        num_steps = run.summary.get('_step')
-        if num_steps is None or num_steps < min_steps:
-            continue 
-        if run.state =='running':
-            frame = load_run(run_path='/'.join(run.path), extractors=EXTRACTORS)
-            csv_path = os.path.join(save_path, f"{run.id}.csv")
-            frame.to_csv(csv_path)
-            
-
-def preload_data():
-    # save all the paths of files to a list in a directory 
-    paths_list = []
-    for path in os.listdir(SAVE_PATH):
-        paths_list.append(os.path.join(SAVE_PATH, path))
-
-    df_list = []
-
-    for path in paths_list:
-        df = pd.read_csv(path,low_memory=False)
-        df_list.append(df)
-        
-    combined_df = pd.concat(df_list, ignore_index=True)
-    return combined_df
+    return df.groupby(entity_choice).I.sum().sort_values().reset_index().tail(ntop)
 
 @st.cache_data()
-def get_metagraph():
+def get_metagraph(time):
+    print(f'Loading metagraph with time {time}')
     subtensor = bt.subtensor(network=NETWORK)
     m = subtensor.metagraph(netuid=NETUID)
     meta_cols = ['I','stake','trust','validator_trust','validator_permit','C','R','E','dividends','last_update']
@@ -341,26 +188,20 @@ def get_metagraph():
     return df_m
 
 
-def load_run(run_path: str, extractors: dict):
+@st.cache_data()
+def load_run(run_path, keys=KEYS):
+
     print('Loading run:', run_path)
     run = api.run(run_path)
-    df = pd.DataFrame(list(run.scan_history()))
-
+    df = pd.DataFrame(list(run.scan_history(keys=keys)))
     for col in ['updated_at', 'best_loss_at', 'created_at']:
         if col in df.columns:
             df[col] = pd.to_datetime(df[col])
-    num_rows=len(df)
-    
-    extractor_df = {key: func(run) for key, func in extractors.items()}
-    repeated_data = {key: [value] * num_rows for key, value in extractor_df.items()}        
-    extractor_df = pd.DataFrame(repeated_data)
-    
-    combined_df = pd.concat([df, extractor_df], axis=1)
-
-    return combined_df
+    print(f'+ Loaded {len(df)} records')
+    return df
 
 @st.cache_data(show_spinner=False)
-def build_data(timestamp=None, paths=BASE_PATH, min_steps=MIN_STEPS, use_cache=True):
+def build_data(timestamp=None, paths=BASE_PATHS, min_steps=MIN_STEPS, use_cache=True):
 
     save_path = '_saved_runs.csv'
     filters = {}
@@ -431,4 +272,10 @@ def load_state_vars():
     }
 
 
+if __name__ == '__main__':
+
+    print('Loading runs')
+    df = load_runs()
 
+    df.to_csv('test_wandb_data.csv', index=False)
+    print(df)