initial dashboard push

README.md

@@ -1,13 +1 @@
----
-title: Sn25
-emoji: 👀
-colorFrom: pink
-colorTo: yellow
-sdk: streamlit
-sdk_version: 1.36.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# folding-api

api.py

@@ -0,0 +1,147 @@
+
+import atexit
+import datetime
+
+from flask import Flask, request, jsonify
+from apscheduler.schedulers.background import BackgroundScheduler
+
+import utils
+
+app = Flask(__name__)
+
+
+# Global variables (saves time on loading data)
+state_vars = None
+reload_timestamp = datetime.datetime.now().strftime('%D %T')
+
+
+def load_data():
+    """
+    Reload the state variables
+    """
+    global state_vars, reload_timestamp
+    state_vars = utils.load_state_vars()
+
+    reload_timestamp = datetime.datetime.now().strftime('%D %T')
+
+    print(f'Reloaded data at {reload_timestamp}')
+
+
+def start_scheduler():
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(func=load_data, trigger="interval", seconds=60*30)
+    scheduler.start()
+
+    # Shut down the scheduler when exiting the app
+    atexit.register(lambda: scheduler.shutdown())
+
+
+@app.route('/', methods=['GET'])
+def home():
+    return "Welcome to the Bittensor Protein Folding Leaderboard API!"
+
+
+@app.route('/updated', methods=['GET'])
+def updated():
+    return reload_timestamp
+
+
+@app.route('/data', methods=['GET'])
+@app.route('/data/<period>', methods=['GET'])
+def data(period=None):
+    """
+    Get the productivity metrics
+    """
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(
+        df.astype(str).to_dict(orient='records')
+    )
+
+@app.route('/productivity', methods=['GET'])
+@app.route('/productivity/<period>', methods=['GET'])
+def productivity_metrics(period=None):
+    """
+    Get the productivity metrics
+    """
+
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(
+        utils.get_productivity(df)
+    )
+
+
+@app.route('/throughput', methods=['GET'])
+@app.route('/throughput/<period>', methods=['GET'])
+def throughput_metrics(period=None):
+    """
+    Get the throughput metrics
+    """
+    assert period in ('24h', None), f"Invalid period: {period}. Must be '24h' or None."
+    df = state_vars["dataframe_24h"] if period == '24h' else state_vars["dataframe"]
+    return jsonify(utils.get_data_transferred(df))
+
+
+@app.route('/metagraph', methods=['GET'])
+def metagraph():
+    """
+    Get the metagraph data
+    Returns:
+    - metagraph_data: List of dicts (from pandas DataFrame)
+    """
+
+    df_m = state_vars["metagraph"]
+
+    return jsonify(
+        df_m.to_dict(orient='records')
+    )
+
+@app.route('/leaderboard', methods=['GET'])
+@app.route('/leaderboard/<entity>', methods=['GET'])
+@app.route('/leaderboard/<entity>/<ntop>', methods=['GET'])
+def leaderboard(entity='identity',ntop=10):
+    """
+    Get the leaderboard data
+    Returns:
+    - leaderboard_data: List of dicts (from pandas DataFrame)
+    """
+
+    assert entity in utils.ENTITY_CHOICES, f"Invalid entity choice: {entity}"
+
+    df_miners = utils.get_leaderboard(
+        state_vars["metagraph"],
+        ntop=int(ntop),
+        entity_choice=entity
+        )
+
+    return jsonify(
+        df_miners.to_dict(orient='records')
+    )
+
+@app.route('/validator', methods=['GET'])
+def validator():
+    """
+    Get the validator data
+    Returns:
+    - validator_data: List of dicts (from pandas DataFrame)
+    """
+    df_m = state_vars["metagraph"]
+    df_validators = df_m.loc[df_m.validator_trust > 0]
+
+    return jsonify(
+        df_validators.to_dict(orient='records')
+    )
+
+
+if __name__ == '__main__':
+
+    load_data()
+    start_scheduler()
+
+    app.run(host='0.0.0.0', port=5001, debug=True)
+    
+
+    # to test locally
+    # curl -X GET http://0.0.0.0:5001/data
+    

app.py

@@ -0,0 +1,140 @@
+import time
+import pandas as pd
+import streamlit as st
+import plotly.express as px
+
+import utils
+
+_ = """
+Proteins folded (delta 24hr)
+Current proteins folding (24hr)
+Average time to fold trend
+Refolded proteins (group by run id and pdb id and get unique)
+Simulation duration distribution
+"""
+
+UPDATE_INTERVAL = 3600
+
+
+st.title('Folding Subnet Dashboard')
+st.markdown('<br>', unsafe_allow_html=True)
+
+# reload data periodically
+df = utils.build_data(time.time()//UPDATE_INTERVAL)
+st.toast(f'Loaded {len(df)} runs')
+
+# TODO: fix the factor for 24 hours ago
+runs_alive_24h_ago = (df.last_event_at > pd.Timestamp.now() - pd.Timedelta('1d'))
+df_24h = df.loc[runs_alive_24h_ago]
+# correction factor to account for the fact that the data straddles the 24h boundary
+# correction factor is based on the fraction of the run which occurred in the last 24h
+# factor = (df_24h.last_event_at - pd.Timestamp.now() + pd.Timedelta('1d')) / pd.Timedelta('1d')
+
+
+#### ------ PRODUCTIVITY ------
+
+# Overview of productivity
+st.subheader('Productivity overview')
+st.info('Productivity metrics show how many proteins have been folded, which is the primary goal of the subnet. Metrics are estimated using weights and biases data combined with heuristics.')
+
+productivity = utils.get_productivity(df)
+productivity_24h = utils.get_productivity(df_24h)
+
+
+m1, m2, m3 = st.columns(3)
+m1.metric('Unique proteins folded', f'{productivity.get("unique_folded"):,.0f}', delta=f'{productivity_24h.get("unique_folded"):,.0f} (24h)')
+m2.metric('Total proteins folded', f'{productivity.get("total_simulations"):,.0f}', delta=f'{productivity_24h.get("total_simulations"):,.0f} (24h)')
+m3.metric('Total simulation steps', f'{productivity.get("total_md_steps"):,.0f}', delta=f'{productivity_24h.get("total_md_steps"):,.0f} (24h)')
+
+st.markdown('<br>', unsafe_allow_html=True)
+
+time_binned_data = df.set_index('last_event_at').groupby(pd.Grouper(freq='12h'))
+
+PROD_CHOICES = {
+    'Unique proteins folded': 'unique_pdbs',
+    'Total simulations': 'total_pdbs',
+    'Total simulation steps': 'total_md_steps',
+}
+prod_choice_label = st.radio('Select productivity metric', list(PROD_CHOICES.keys()), index=0, horizontal=True)
+prod_choice = PROD_CHOICES[prod_choice_label]
+steps_running_total = time_binned_data[prod_choice].sum().cumsum()
+st.plotly_chart(
+    # add fillgradient to make it easier to see the trend
+    px.area(steps_running_total, y=prod_choice, 
+            labels={'last_event_at':'', prod_choice: prod_choice_label},
+    ).update_traces(fill='tozeroy'),
+    use_container_width=True,
+)
+
+st.markdown('<br>', unsafe_allow_html=True)
+
+
+#### ------ THROUGHPUT ------
+st.subheader('Throughput overview')
+
+st.info('Throughput metrics show the total amount of data sent and received by the validators. This is a measure of the network activity and the amount of data that is being processed by the subnet.')
+
+MEM_UNIT = 'GB' #st.radio('Select memory unit', ['TB','GB', 'MB'], index=0, horizontal=True)
+
+data_transferred = utils.get_data_transferred(df,unit=MEM_UNIT)
+data_transferred_24h = utils.get_data_transferred(df_24h, unit=MEM_UNIT)
+
+m1, m2, m3 = st.columns(3)
+m1.metric(f'Total sent data ({MEM_UNIT})', f'{data_transferred.get("sent"):,.0f}', delta=f'{data_transferred_24h.get("sent"):,.0f} (24h)')
+m2.metric(f'Total received data ({MEM_UNIT})', f'{data_transferred.get("received"):,.0f}', delta=f'{data_transferred_24h.get("received"):,.0f} (24h)')
+m3.metric(f'Total transferred data ({MEM_UNIT})', f'{data_transferred.get("total"):,.0f}', delta=f'{data_transferred_24h.get("total"):,.0f} (24h)')
+
+
+IO_CHOICES = {'total_data_sent':'Sent', 'total_data_received':'Received'}
+io_running_total = time_binned_data[list(IO_CHOICES.keys())].sum().rename(columns=IO_CHOICES).cumsum().melt(ignore_index=False)
+io_running_total['value'] = io_running_total['value'].apply(utils.convert_unit, args=(utils.BASE_UNITS, MEM_UNIT))
+
+st.plotly_chart(
+    px.area(io_running_total, y='value', color='variable',
+            labels={'last_event_at':'', 'value': f'Data transferred ({MEM_UNIT})', 'variable':'Direction'},
+    ),
+    use_container_width=True,
+)
+
+st.markdown('<br>', unsafe_allow_html=True)
+
+
+#### ------ LEADERBOARD ------
+
+st.subheader('Leaderboard')
+st.info('The leaderboard shows the top miners by incentive.')
+m1, m2 = st.columns(2)
+ntop = m1.slider('Number of top miners to display', value=10, min_value=3, max_value=50, step=1)
+entity_choice = m2.radio('Select entity', utils.ENTITY_CHOICES, index=0, horizontal=True)
+
+df_m = utils.get_metagraph(time.time()//UPDATE_INTERVAL)
+df_miners = utils.get_leaderboard(df_m, ntop=ntop, entity_choice=entity_choice)
+
+# hide colorbar and don't show y axis
+st.plotly_chart(
+    px.bar(df_miners, x='I', color='I', hover_name=entity_choice, text=entity_choice if ntop < 20 else None,
+            labels={'I':'Incentive', 'trust':'Trust', 'stake':'Stake', '_index':'Rank'},
+    ).update_layout(coloraxis_showscale=False, yaxis_visible=False),
+    use_container_width=True,
+)
+
+
+with st.expander('Show raw metagraph data'):
+    st.dataframe(df_m)
+
+st.markdown('<br>', unsafe_allow_html=True)
+
+
+#### ------ LOGGED RUNS ------
+
+st.subheader('Logged runs')
+st.info('The timeline shows the creation and last event time of each run.')
+st.plotly_chart(
+    px.timeline(df, x_start='created_at', x_end='last_event_at', y='username', color='state',
+                labels={'created_at':'Created at', 'last_event_at':'Last event at', 'username':''},
+                ),
+    use_container_width=True
+)
+
+with st.expander('Show raw run data'):
+    st.dataframe(df)

requirements.txt

@@ -0,0 +1,11 @@
+bittensor
+requests
+wandb
+python-dotenv
+APScheduler
+streamlit
+nbformat
+plotly
+pandas
+flask
+

utils.py

@@ -0,0 +1,270 @@
+import os
+import tqdm
+import time
+import wandb
+import streamlit as st
+import pandas as pd
+import bittensor as bt
+
+
+# TODO: Store the runs dataframe (as in sn1 dashboard) and top up with the ones created since the last snapshot
+# TODO: Store relevant wandb data in a database for faster access
+
+# TODO: filter out netuid 141(?)
+
+MIN_STEPS = 12 # minimum number of steps in wandb run in order to be worth analyzing
+MAX_RUNS = 100#0000
+NETUID = 25
+BASE_PATH = 'opentensor-dev/folding-validators'
+NETWORK = 'finney'
+KEYS = None
+ABBREV_CHARS = 8
+ENTITY_CHOICES = ('identity', 'hotkey', 'coldkey')
+
+PDBS_PER_RUN_STEP = 0.083
+AVG_MD_STEPS = 30_000
+BASE_UNITS = 'MB'
+
+api = wandb.Api(timeout=120)
+
+IDENTITIES = {
+    '5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3': 'opentensor',
+    '5Hddm3iBFD2GLT5ik7LZnT3XJUnRnN8PoeCFgGQgawUVKNm8': 'taostats',
+    '5HEo565WAy4Dbq3Sv271SAi7syBSofyfhhwRNjFNSM2gP9M2': 'foundry',
+    '5HK5tp6t2S59DywmHRWPBVJeJ86T61KjurYqeooqj8sREpeN': 'bittensor-guru',
+    '5FFApaS75bv5pJHfAp2FVLBj9ZaXuFDjEypsaBNc1wCfe52v': 'roundtable-21',
+    '5EhvL1FVkQPpMjZX4MAADcW42i3xPSF1KiCpuaxTYVr28sux': 'tao-validator',
+    '5FKstHjZkh4v3qAMSBa1oJcHCLjxYZ8SNTSz1opTv4hR7gVB': 'datura',
+    '5DvTpiniW9s3APmHRYn8FroUWyfnLtrsid5Mtn5EwMXHN2ed': 'first-tensor',
+    '5HbLYXUBy1snPR8nfioQ7GoA9x76EELzEq9j7F32vWUQHm1x': 'tensorplex',
+    '5CsvRJXuR955WojnGMdok1hbhffZyB4N5ocrv82f3p5A2zVp': 'owl-ventures',
+    '5CXRfP2ekFhe62r7q3vppRajJmGhTi7vwvb2yr79jveZ282w': 'rizzo',
+    '5HNQURvmjjYhTSksi8Wfsw676b4owGwfLR2BFAQzG7H3HhYf': 'neural-internet'
+}
+
+EXTRACTORS = {
+    'state': lambda x: x.state,
+    'run_id': lambda x: x.id,
+    'user': lambda x: x.user.name[:16],
+    'username': lambda x: x.user.username[:16],
+    'created_at': lambda x: pd.Timestamp(x.created_at),
+    'last_event_at': lambda x: pd.Timestamp(x.summary.get('_timestamp'), unit='s'),
+
+    'netuid': lambda x: x.config.get('netuid'),
+    'mock': lambda x: x.config.get('neuron').get('mock'),
+    'sample_size': lambda x: x.config.get('neuron').get('sample_size'),
+    'queue_size': lambda x: x.config.get('neuron').get('queue_size'),
+    'timeout': lambda x: x.config.get('neuron').get('timeout'),
+    'update_interval': lambda x: x.config.get('neuron').get('update_interval'),
+    'epoch_length': lambda x: x.config.get('neuron').get('epoch_length'),
+    'disable_set_weights': lambda x: x.config.get('neuron').get('disable_set_weights'),
+
+    # This stuff is from the last logged event
+    'num_steps': lambda x: x.summary.get('_step'),
+    'runtime': lambda x: x.summary.get('_runtime'),
+    'init_energy': lambda x: x.summary.get('init_energy'),
+    'best_energy': lambda x: x.summary.get('best_loss'),
+    'pdb_id': lambda x: x.summary.get('pdb_id'),
+    'pdb_updates': lambda x: x.summary.get('updated_count'),
+    'total_returned_sizes': lambda x: get_total_file_sizes(x),
+    'total_sent_sizes': lambda x: get_total_md_input_sizes(x),
+
+    'pdb_atoms': lambda x: get_pdb_complexity(x),
+
+    'version': lambda x: x.tags[0],
+    'spec_version': lambda x: x.tags[1],
+    'vali_hotkey': lambda x: x.tags[2],
+
+    # System metrics
+    'disk_read': lambda x: x.system_metrics.get('system.disk.in'),
+    'disk_write': lambda x: x.system_metrics.get('system.disk.out'),
+    # Really slow stuff below
+    # 'started_at': lambda x: x.metadata.get('startedAt'),
+    # 'disk_used': lambda x: x.metadata.get('disk').get('/').get('used'),
+    # 'commit': lambda x: x.metadata.get('git').get('commit')
+}
+
+def get_pdb_complexity(run, field='ATOM', preprocess=True):
+    data = run.summary.get('pdb_complexity')
+
+    if not isinstance(data, list) or len(data)==0:
+        return None
+    data = data[0]
+
+    counts = data.get(field)
+    if counts is not None:
+        return counts
+
+    counts = 0
+    for key in data.keys():
+        if key.startswith(field):
+            counts+=data.get(key)
+    return counts
+
+def convert_unit(value, from_unit, to_unit):
+    """Converts a value from one unit to another
+
+    example:
+    convert_unit(1024, 'KB', 'MB') -> 1
+    convert_unit(1024, 'MB', 'KB') -> 1048576
+    """
+    units = ['B', 'KB','MB','GB','TB']
+    assert from_unit.upper() in units, f'From unit {from_unit!r} not in {units}'
+    assert to_unit.upper() in units, f'To unit {to_unit!r} not in {units}'
+
+    factor = 1024**(units.index(from_unit) - units.index(to_unit))
+    # print(f'Converting from {from_unit!r} to {to_unit!r}, factor: {factor}')
+    return value * factor
+
+def get_total_file_sizes(run):
+    """returns total size of byte strings in bytes"""
+    size_bytes = sum(size for sizes in run.summary.get('response_returned_files_sizes',[[]]) for size in sizes if sizes)
+    return convert_unit(size_bytes, from_unit='B', to_unit=BASE_UNITS)
+
+def get_total_md_input_sizes(run):
+    """returns total size of byte strings in bytes"""
+    size_bytes = sum(run.summary.get('md_inputs_sizes',[]))
+    return convert_unit(size_bytes, from_unit='B', to_unit=BASE_UNITS)
+
+
+def get_data_transferred(df, unit='GB'):
+
+    factor = convert_unit(1, from_unit=BASE_UNITS, to_unit=unit)
+    sent = df.total_data_sent.sum()
+    received = df.total_data_received.sum()
+    return {
+        'sent':sent * factor,
+        'received':received * factor,
+        'total': (sent + received) * factor,
+        'read':df.disk_read.sum() * factor,
+        'write':df.disk_write.sum() * factor,
+        }
+
+
+def get_productivity(df):
+
+    # Estimate the number of unique pdbs folded using our heuristic
+    unique_folded = df.unique_pdbs.sum().round()
+    # Estimate the total number of simulations completed using our heuristic
+    total_simulations = df.total_pdbs.sum().round()
+    # Estimate the total number of simulation steps completed using our heuristic
+    total_md_steps = df.total_md_steps.sum().round()
+
+    return {
+        'unique_folded': unique_folded,
+        'total_simulations': total_simulations,
+        'total_md_steps': total_md_steps,
+    }
+
+def get_leaderboard(df, ntop=10, entity_choice='identity'):
+
+    df = df.loc[df.validator_permit==False]
+    df.index = range(df.shape[0])
+    return df.groupby(entity_choice).I.sum().sort_values().reset_index().tail(ntop)
+
+@st.cache_data()
+def get_metagraph(time):
+    print(f'Loading metagraph with time {time}')
+    subtensor = bt.subtensor(network=NETWORK)
+    m = subtensor.metagraph(netuid=NETUID)
+    meta_cols = ['I','stake','trust','validator_trust','validator_permit','C','R','E','dividends','last_update']
+
+    df_m = pd.DataFrame({k: getattr(m, k) for k in meta_cols})
+    df_m['uid'] = range(m.n.item())
+    df_m['hotkey'] = list(map(lambda a: a.hotkey, m.axons))
+    df_m['coldkey'] = list(map(lambda a: a.coldkey, m.axons))
+    df_m['ip'] = list(map(lambda a: a.ip, m.axons))
+    df_m['port'] = list(map(lambda a: a.port, m.axons))
+    df_m['coldkey'] = df_m.coldkey.str[:ABBREV_CHARS]
+    df_m['hotkey'] = df_m.hotkey.str[:ABBREV_CHARS]
+    df_m['identity'] = df_m.apply(lambda x: f'{x.hotkey} @ uid {x.uid}', axis=1)
+    return df_m
+
+
+@st.cache_data()
+def load_run(run_path, keys=KEYS):
+
+    print('Loading run:', run_path)
+    run = api.run(run_path)
+    df = pd.DataFrame(list(run.scan_history(keys=keys)))
+    for col in ['updated_at', 'best_loss_at', 'created_at']:
+        if col in df.columns:
+            df[col] = pd.to_datetime(df[col])
+    print(f'+ Loaded {len(df)} records')
+    return df
+
+@st.cache_data(show_spinner=False)
+def build_data(timestamp=None, path=BASE_PATH, min_steps=MIN_STEPS, use_cache=True):
+
+    save_path = '_saved_runs.csv'
+    filters = {}
+    df = pd.DataFrame()
+    # Load the last saved runs so that we only need to update the new ones
+    if use_cache and os.path.exists(save_path):
+        df = pd.read_csv(save_path)
+        df['created_at'] = pd.to_datetime(df['created_at'])
+        df['last_event_at'] = pd.to_datetime(df['last_event_at'])
+
+        timestamp_str = df['last_event_at'].max().isoformat()
+        filters.update({'updated_at': {'$gte': timestamp_str}})
+
+    progress = st.progress(0, text='Loading data')
+
+    runs = api.runs(path, filters=filters)
+
+    run_data = []
+    n_events = 0
+    for i, run in enumerate(tqdm.tqdm(runs, total=len(runs))):
+        num_steps = run.summary.get('_step',0)
+        if num_steps<min_steps:
+            continue
+        n_events += num_steps
+        prog_msg = f'Loading data {i/len(runs)*100:.0f}%, {n_events:,.0f} events)'
+        progress.progress(i/len(runs),text=f'{prog_msg}... **downloading** `{os.path.join(*run.path)}`')
+
+        run_data.append(run)
+
+    progress.empty()
+
+    df_new = pd.DataFrame([{k: func(run) for k, func in EXTRACTORS.items()} for run in tqdm.tqdm(run_data, total=len(run_data))])
+    df = pd.concat([df, df_new], ignore_index=True)
+    df['duration'] = (df.last_event_at - df.created_at).round('s')
+    df['identity'] = df['vali_hotkey'].map(IDENTITIES).fillna('unknown')
+    df['vali_hotkey'] = df['vali_hotkey'].str[:ABBREV_CHARS]
+
+    # Estimate the number of unique pdbs in a run as a function of the steps in the run
+    df['unique_pdbs'] = df['num_steps'] * PDBS_PER_RUN_STEP
+    df['total_pdbs'] = df['unique_pdbs'] * df['sample_size']
+    # Estimate the number of md steps as the average per simulation multiplied by our estimate of total sims
+    df['total_md_steps'] = df['total_pdbs'] * AVG_MD_STEPS
+
+    df['total_data_sent'] = df['total_sent_sizes'] * df['num_steps']
+    df['total_data_received'] = df['total_returned_sizes'] * df['num_steps']
+
+    df.to_csv(save_path, index=False)
+    return df
+
+
+def load_state_vars():
+    UPDATE_INTERVAL = 600
+
+    df = build_data(time.time()//UPDATE_INTERVAL)
+    runs_alive_24h_ago = (df.last_event_at > pd.Timestamp.now() - pd.Timedelta('1d'))
+    df_24h = df.loc[runs_alive_24h_ago]
+
+    df_m = get_metagraph(time.time()//UPDATE_INTERVAL)
+
+    return {
+        'dataframe': df,
+        'dataframe_24h': df_24h,
+        'metagraph': df_m,
+    }
+
+
+if __name__ == '__main__':
+
+    print('Loading runs')
+    df = load_runs()
+
+    df.to_csv('test_wandb_data.csv', index=False)
+    print(df)