renw

app.py

@@ -1,239 +1,158 @@
-import streamlit as st
-import wandb
-import pandas as pd
-import os
-import time
-import datetime
-from utils import fetch_competition_summary, fetch_models_evaluation, get_all_competition_summary, set_custom_table_style, highlight_score_column
-
-st.set_page_config(layout="wide")
-
-table_font_size = 16
-set_custom_table_style()
-
-### WANDB
-
-# Access the API key from the environment variable
-wandb_api_key = os.getenv('WANDB_API_KEY')
-
-# Log in to wandb using the API key
-if wandb_api_key:
-    wandb.login(key=wandb_api_key)
-else:
-    st.error("WANDB_API_KEY not found in environment variables.")
-
-# Initialize W&B API
-api = wandb.Api()
-
-### STREAMLIT APP
-
-# Define available projects (bookmarks)
-projects = {
-    "Melanoma-1": {
-        "entity": "safe-scan-ai",
-        "project": "melanoma-1",
-        "description": "This is a testnet competition for melanoma detection."
-    },
-    "Melanoma-testnet": {
-        "entity": "safe-scan-ai",
-        "project": "melanoma-testnet",
-        "description": "This is a testnet competition for melanoma detection."
-    },
-    # Add more projects as needed
-}
-
-# Initialize leader info
-@st.cache_data
-def update_leader_info(leader_info, competition, best_model):
-    if leader_info.get(competition) is None:
-        leader_info[competition] = {
-            "Username": best_model["Username"],
-            "Hotkey": best_model["Hotkey"],
-            "Date": time.strftime("%Y-%m-%d"),
-            "Days on Top": 1
-        }
-    else:
-        if leader_info[competition]["Hotkey"] == best_model["Hotkey"]:
-            # count the number of days on top
-            start_date = datetime.datetime.strptime(leader_info[competition]["Date"], "%Y-%m-%d")
-            current_date = datetime.datetime.now()
-            days_on_top = (current_date - start_date).days
-            leader_info[competition]["Days on Top"] = days_on_top + 1
-        else:
-            leader_info[competition]["Username"] = best_model["Username"]
-            leader_info[competition]["Hotkey"] = best_model["Hotkey"]
-            leader_info[competition]["Date"] = time.strftime("%Y-%m-%d")
-            leader_info[competition]["Days on Top"] = 1
-    return leader_info[competition]
-
-@st.cache_data()
-def load_competition_data(last_update_time=None):
-    competition_summaries = {}
-    model_evaluations = {}
-    for competition in projects:
-        competition_summaries[competition] = fetch_competition_summary(api, projects, competition)
-        model_evaluations[competition] = fetch_models_evaluation(api, projects, competition)
-    
-    last_update_time = time.time()
-    return competition_summaries, model_evaluations, last_update_time
-
-competition_draw = {}
-
-# Streamlit app main function
-def main():
-    st.markdown("<h1 style='text-align: center;'>SafeScan Dashboard</h1>", unsafe_allow_html=True)
-    st.markdown("<h3 style='text-align: center;'>Welcome to the Competition Dashboard!</h3>", unsafe_allow_html=True)
-    st.write("Explore the various AI competitions and their respective rankings. Select a competition to view more details and rankings.")
-
-    # Add links to important resources
-    st.markdown("### Important Links")
-    st.markdown("""
-    - [Official Subnet Repository](https://github.com/safe-scan-ai/cancer-ai)
-    - [Discord Community](https://discord.com/invite/sFyJ35yQ)
-    - [Project Website](https://safe-scan.ai/)
-    - [Mobile App Website](https://skin-scan.ai/)
-    """)
-
-    update_interval = 2 * 60  # 20 minutes in seconds
-
-    if 'last_update_time' not in st.session_state:
-        st.session_state.last_update_time = None
-    if "leader_info" not in st.session_state:
-        st.session_state.leader_info = {}
-    if 'selected_competition' not in st.session_state:
-        st.session_state.selected_competition = None
-
-    if st.session_state.last_update_time is None or (time.time() - st.session_state.last_update_time > update_interval):
-        competition_summaries, model_evaluations, st.session_state.last_update_time = load_competition_data(st.session_state.last_update_time)
-
-        for competition in projects:
-            if not competition_summaries[competition].empty:
-                # get all winning hotkeys and number of wins
-                winning_hotkeys = competition_summaries[competition]["Winning Hotkey"].value_counts()
-
-                # if not empty, get the best hotkey
-                if not winning_hotkeys.empty:
-                    # Get the number of model with the same value
-                    if winning_hotkeys.value_counts().max() > 1:
-                        # add all the models with the with max numbers of wins
-                        hotkeys = winning_hotkeys[winning_hotkeys == winning_hotkeys.max()].index
-                        # change the info in competition_draw
-                        competition_draw[competition] = hotkeys
-                    else:
-                        # remove entries in competition_draw
-                        competition_draw.pop(competition, None)
-                    
-
-                    best_hotkey = winning_hotkeys.idxmax()
-
-                    # Filter models for the best hotkey
-                    best_model_filtered = model_evaluations[competition][model_evaluations[competition]["Hotkey"] == best_hotkey]
-
-                    # Check if the filtered DataFrame is not empty
-                    if not best_model_filtered.empty:
-                        best_model = best_model_filtered.iloc[0]
-                        st.session_state.leader_info[competition] = update_leader_info(st.session_state.leader_info, competition, best_model)
-                    else:
-                        st.warning(f"No model found for the best hotkey: {best_hotkey} in competition {competition}.")
-
-            else:
-                st.session_state.leader_info[competition] = {
-                    "Username": "N/A", "Hotkey": "N/A",
-                    "Date": "N/A", "UID": "N/A", "Days on Top": "N/A"
-                }
-    else:
-        competition_summaries, model_evaluations, _ = load_competition_data(st.session_state.last_update_time)
-
-    # display ifo abut draw
-    if len(competition_draw) > 0:
-        st.write("Draws in the competition")
-        for competition, hotkeys in competition_draw.items():
-            st.write(f"Competition: {competition}")
-            for hotkey in hotkeys:
-                st.write(f"Hotkey: {hotkey}")
-
-
-    number_of_competitions, number_of_runs = get_all_competition_summary(api, projects)
-    last_updated = datetime.datetime.fromtimestamp(st.session_state.last_update_time).strftime('%Y-%m-%d %H:%M:%S')
-
-    # Set the desired font size here (in pixels)
-    summary_font_size = 14  # You can adjust this value as needed
-
-    st.subheader("Dashboard summary information")
-    
-    
-    # Display summary statistics in a table-like format
-    st.markdown(f"""
-    <div class="table-container">
-        <table class="summary-table">
-            <tr>
-                <th>Number of competitions</th>
-                <th>Number of models run</th>
-                <th>Last updated</th>
-            </tr>
-            <tr>
-                <td>{number_of_competitions}</td>
-                <td>{number_of_runs}</td>
-                <td>{last_updated}</td>
-            </tr>
-        </table>
-    </div>
-    """, unsafe_allow_html=True)
-
-
-    
-    st.subheader("###")
-    st.markdown("<h2 style='text-align: center; font-size: 28px;'>Competitions</h2>", unsafe_allow_html=True)
-    st.write("#### Select a competition to view more details and rankings.")
-
-    # Create a header for the table
-    cols = st.columns([1, 3, 2, 2, 2, 1, 2])
-    cols[0].write("Index")
-    cols[1].write("Competition Name")
-    cols[2].write("Leader")
-    cols[3].write("Date")
-    cols[4].write("Hotkey")
-    cols[5].write("Days on Top")
-
-    for index, (competition, details) in enumerate(projects.items(), start=1):
-        leader_info = st.session_state.get("leader_info", {}).get(competition, {})
-
-        cols = st.columns([1, 3, 2, 2, 2, 1, 2])
-        cols[0].write(index)
-        if cols[1].button(competition):
-            st.session_state.selected_competition = competition
-        cols[2].write(leader_info.get("Username", "N/A"))
-        cols[3].write(leader_info.get("Date", "N/A"))
-        cols[4].write(leader_info.get("Hotkey", "N/A"))
-        cols[5].write(leader_info.get("Days on Top", "N/A"))
-    if st.session_state.selected_competition:
-        competition_name = st.session_state.selected_competition
-        competition_details = projects.get(competition_name, {})
-        description = competition_details.get("description", "No description available.")
-        
-        st.subheader(f"Competition: {competition_name}")
-        st.write(description)
-
-        st.subheader("Competition Summary")
-        competition_summary_df = competition_summaries.get(competition_name, pd.DataFrame())
-
-        if not competition_summary_df.empty:
-            st.dataframe(competition_summary_df)
-        else:
-            st.warning("No competition summary data available.")
-
-
-        # TODO highlight score collumn
-        st.subheader("Models Evaluation")
-        models_evaluation_df = model_evaluations.get(competition_name, pd.DataFrame())
-        if not models_evaluation_df.empty:
-            st.dataframe(models_evaluation_df.style.apply(highlight_score_column, axis=0))
-        else:
-            st.warning("No models evaluation data available.")
-    else:
-        st.write("Please select a competition to view details.")
-
-
-# Run the app
-if __name__ == "__main__":
+import streamlit as st
+import wandb
+import pandas as pd
+import os
+import time
+import datetime
+from utils import fetch_competition_summary, fetch_models_evaluation, get_all_competition_summary, update_leader_info
+from dotenv import load_dotenv
+from cfg import * 
+load_dotenv()
+
+st.set_page_config(layout="wide")
+
+table_font_size = 16
+st.markdown(TABLE_STYLE, unsafe_allow_html=True)
+
+### WANDB
+
+# Access the API key from the environment variable
+wandb_api_key = os.getenv('WANDB_API_KEY')
+
+# Log in to wandb using the API key
+if wandb_api_key:
+    wandb.login(key=wandb_api_key)
+    # wandb.login()
+else:
+    st.error("WANDB_API_KEY not found in environment variables.")
+
+wandb_api = wandb.Api()
+
+
+
+@st.cache_data()
+def load_competition_data(last_update_time=None):
+    competition_summaries = {}
+    model_evaluations = {}
+    for competition in COMPETITIONS:
+        competition_summaries[competition] = fetch_competition_summary(wandb_api, COMPETITIONS, competition)
+        model_evaluations[competition] = fetch_models_evaluation(wandb_api, COMPETITIONS, competition)
+    
+    last_update_time = time.time()
+    return competition_summaries, model_evaluations, last_update_time
+
+# Streamlit app main function
+def main():
+    st.markdown(HEADER, unsafe_allow_html=True)
+
+    if 'last_update_time' not in st.session_state:
+        st.session_state.last_update_time = None
+    if "leader_info" not in st.session_state:
+        st.session_state.leader_info = {}
+    if 'selected_competition' not in st.session_state:
+        st.session_state.selected_competition = None
+
+    if st.session_state.last_update_time is None or (time.time() - st.session_state.last_update_time > UPDATE_INTERVAL):
+        competition_summaries, model_evaluations, st.session_state.last_update_time = load_competition_data(st.session_state.last_update_time)
+
+        for competition in COMPETITIONS:
+            if not competition_summaries[competition].empty:
+                # get all winning hotkeys and number of wins
+                winning_hotkeys = competition_summaries[competition]["Winning Hotkey"].value_counts()
+
+                # if not empty, get the best hotkey
+                if not winning_hotkeys.empty:
+                    best_hotkey = winning_hotkeys.idxmax()
+
+                    # get the best model with the winning hotkey
+                    best_model = model_evaluations[competition][model_evaluations[competition]["Hotkey"] == best_hotkey].iloc[0]
+                    st.session_state.leader_info[competition] = update_leader_info(st.session_state.leader_info, competition, best_model)
+            else:
+                st.session_state.leader_info[competition] = {
+                    "Username": "N/A", "Model Name": "N/A", "Hotkey": "N/A",
+                    "Date": "N/A", "UID": "N/A", "Days on Top": "N/A"
+                }
+    else:
+        competition_summaries, model_evaluations, _ = load_competition_data(st.session_state.last_update_time)
+
+    number_of_competitions, number_of_runs = get_all_competition_summary(wandb_api, COMPETITIONS)
+    last_updated = datetime.datetime.fromtimestamp(st.session_state.last_update_time).strftime('%Y-%m-%d %H:%M:%S')
+
+    # Set the desired font size here (in pixels)
+    summary_font_size = 14  # You can adjust this value as needed
+
+    st.subheader("###")
+    st.markdown("<h2 style='text-align: center; font-size: 28px;'>Competitions</h2>", unsafe_allow_html=True)
+    st.write("#### Select a competition to view more details and rankings.")
+    
+    # Create a header for the table
+    cols = st.columns([1, 3, 2, 2, 2, 2, 1, 2])
+    headers = ["Index", "Competition Name", "Leader", "Model", "Date", "Hotkey", "Days on Top"]
+    for col, header in zip(cols, headers):
+        col.write(header)
+
+    
+    for index, (competition, details) in enumerate(COMPETITIONS.items(), start=1):
+        leader_info = st.session_state.get("leader_info", {}).get(competition, {})
+
+        cols = st.columns([1, 3, 2, 2, 2, 2, 1, 2])
+        cols[0].write(index)
+
+        if cols[1].button(competition):
+            st.session_state.selected_competition = competition
+        cols[2].write(leader_info.get("Username", "N/A"))
+        cols[3].write(leader_info.get("Model Name", "N/A"))
+        cols[4].write(leader_info.get("Date", "N/A"))
+        cols[5].write(leader_info.get("Hotkey", "N/A"))
+        cols[6].write(leader_info.get("Days on Top", "N/A"))
+    if st.session_state.selected_competition:
+        competition_name = st.session_state.selected_competition
+        competition_details = COMPETITIONS.get(competition_name, {})
+        description = competition_details.get("description", "No description available.")
+        
+        st.subheader(f"Competition: {competition_name}")
+        st.write(description)
+
+        st.subheader("Competition Summary")
+        competition_summary_df = competition_summaries.get(competition_name, pd.DataFrame())
+        if not competition_summary_df.empty:
+            st.markdown('<div class="table-container">' + competition_summary_df.to_html(classes='dataframe', escape=False) + '</div>', unsafe_allow_html=True)
+        else:
+            st.warning("No competition summary data available.")
+
+        st.subheader("Models Evaluation")
+        models_evaluation_df = model_evaluations.get(competition_name, pd.DataFrame())
+        if not models_evaluation_df.empty:
+            st.markdown('<div class="table-container">' + models_evaluation_df.to_html(classes='dataframe', escape=False) + '</div>', unsafe_allow_html=True)
+        else:
+            st.warning("No models evaluation data available.")
+    else:
+        st.write("Please select a competition to view details.")
+
+    
+    
+    st.subheader("Summary information")
+    st.markdown(f"""
+    <div class="table-container">
+        <table class="summary-table">
+            <tr>
+                <th>Number of competitions</th>
+                <th>Number of models run</th>
+                <th>Last updated</th>
+            </tr>
+            <tr>
+                <td>{number_of_competitions}</td>
+                <td>{number_of_runs}</td>
+                <td>{last_updated}</td>
+            </tr>
+        </table>
+    </div>
+    """, unsafe_allow_html=True)
+
+
+    
+
+
+# Run the app
+if __name__ == "__main__":
     main()

cfg.py

@@ -0,0 +1,74 @@
+UPDATE_INTERVAL = 2 * 60  # 20 minutes in seconds
+
+
+
+COMPETITIONS = {
+    "Melanoma-1": {
+        "entity": "safe-scan-ai",
+        "project": "melanoma-1",
+        "description": "This is a testnet competition for melanoma detection."
+    },
+    "Melanoma-testnet": {
+        "entity": "safe-scan-ai",
+        "project": "melanoma-testnet",
+        "description": "This is a testnet competition for melanoma detection."
+    },
+    
+    # Add more projects as needed
+}
+
+
+
+HEADER = """
+<h1 style='text-align: center;'>Safe Scan competition dashboard</h1>
+<a href='https://safe-scan.ai/'><img src='https://img.shields.io/badge/Website-Project-blue?logo=google-chrome' alt='Safe Scan website'></a>
+<a href='https://github.com/safe-scan-ai/cancer-ai'><img src='https://img.shields.io/badge/GitHub-Repository-blue?logo=github' alt='GitHub'></a>
+<a href='https://discord.com/invite/sFyJ35yQ'><img src='https://img.shields.io/badge/Discord-Community-blue?logo=discord' alt='Discord'></a>
+"""
+
+
+
+
+TABLE_STYLE = """
+    <style>
+    .dataframe, .summary-table {
+        width: 100%;
+        border-collapse: collapse;
+        margin: 20px 0;
+        font-size: 16px;
+        min-width: 400px;
+    }
+    .dataframe th,
+    .dataframe td,
+    .summary-table th,
+    .summary-table td {
+        padding: 12px 15px;
+        border: 1px solid #CCCCCC;
+        text-align: left;
+    }
+    .dataframe thead th,
+    .summary-table th {
+        background-color: #333333;
+        color: #FFFFFF;
+    }
+    .dataframe tbody tr:nth-child(even),
+    .summary-table tr:nth-child(even) {
+        background-color: #F0F0F0;
+    }
+    .dataframe tbody tr:nth-child(odd),
+    .summary-table tr:nth-child(odd) {
+        background-color: #E0E0E0;
+    }
+    .dataframe tbody td,
+    .summary-table td {
+        color: #333333;
+    }
+    .dataframe tbody tr:hover,
+    .summary-table tr:hover {
+        background-color: #D1D1D1;
+    }
+    .table-container {
+        overflow-x: auto;
+    }
+    </style>
+    """

utils.py

@@ -1,7 +1,32 @@
 import streamlit as st
 import pandas as pd
 import wandb
-import datetime
+from datetime import datetime
+
+
+
+def update_leader_info(leader_info, competition, best_model):
+    current_date = datetime.now().strftime("%Y-%m-%d")
+    if leader_info.get(competition) is None:
+        leader_info[competition] = {
+            "Username": best_model["Username"],
+            "Model Name": best_model["Model Name"],
+            "Hotkey": best_model["Hotkey"],
+            "Date": current_date,
+            "Days on Top": 1
+        }
+    else:
+        if leader_info[competition]["UID"] == best_model["ID"]:
+            leader_info[competition]["Days on Top"] += 1
+        else:
+            leader_info[competition] = {
+                "Username": best_model["Username"],
+                "Model Name": best_model["Model Name"],
+                "Hotkey": best_model["Hotkey"],
+                "Date": current_date,
+                "Days on Top": 1
+            }
+    return leader_info[competition]
 
 
 def get_all_competition_summary(api, projects):
@@ -29,10 +54,10 @@ def fetch_competition_summary(api, projects, selected_project):
     for run in runs:
         try:
             summary = run.summary
-            if summary.get("validator_hotkey") and summary.get("winning_hotkey"):
+            if summary.get("validator_id") and summary.get("winning_hotkey"):
                 data.append({
                     "ID": run.id,
-                    "Validator Hotkey": summary.get("validator_hotkey"),
+                    "Validator ID": summary.get("validator_id"),
                     "Winning Hotkey": summary.get("winning_hotkey"),
                     "Run Time (s)": summary.get("run_time_s"),
                     "Created At": run.created_at,
@@ -58,25 +83,18 @@ def fetch_models_evaluation(api, projects, selected_project):
     for run in runs:
         try:
             summary = run.summary
-            if summary.get("score") is not None:  # Assuming runs with score are model evaluations
+            if summary.get("accuracy") is not None:  # Assuming runs with accuracy are model evaluations
                 data.append({
                     "ID": run.id,
                     "Username": run.user.username,
-                    "Hotkey": summary.get("miner_hotkey", "N/A"),
+                    "Model Name": summary.get("model_name", "N/A"),
+                    "Hotkey": summary.get("hotkey", "N/A"),
                     "Score": summary.get("score"),
-
-                    "F1-beta": summary.get("fbeta"),
+                    "F1-beta": summary.get("f1-beta"),
                     "Accuracy": summary.get("accuracy"),
                     "Recall": summary.get("recall"),
-                    "Precision": summary.get("precision"),
-
-                    "Tested entries": summary.get("tested_entries"),
-
                     "ROC AUC": summary.get("roc_auc"),
-                    "Confusion Matrix": summary.get("confusion_matrix"),
-
                     "Created At": run.created_at,
-                    #TODO link to huggingface model
                 })
         except Exception as e:
             st.write(f"Error processing run {run.id}: {str(e)}")
@@ -88,74 +106,3 @@ def fetch_models_evaluation(api, projects, selected_project):
     
     return df
 
-def update_leader_info(leader_info, competition, best_model):
-    current_date = datetime.now().strftime("%Y-%m-%d")
-    if leader_info.get(competition) is None:
-        leader_info[competition] = {
-            "Username": best_model["Username"],
-            "Hotkey": best_model["Hotkey"],
-            "Date": current_date,
-            "Days on Top": 1
-        }
-    else:
-        if leader_info[competition]["UID"] == best_model["ID"]:
-            leader_info[competition]["Days on Top"] += 1
-        else:
-            leader_info[competition] = {
-                "Username": best_model["Username"],
-                "Hotkey": best_model["Hotkey"],
-                "Date": current_date,
-                "Days on Top": 1
-            }
-    return leader_info[competition]
-
-def highlight_score_column(s):
-    """
-    Highlight the 'Score' column with a custom background color.
-    """
-    return ['background-color: yellow' if s.name == 'Score' else '' for _ in s]
-
-def set_custom_table_style():
-    st.markdown("""
-    <style>
-    .dataframe, .summary-table {
-        width: 100%;
-        border-collapse: collapse;
-        margin: 20px 0;
-        font-size: 16px;
-        min-width: 400px;
-    }
-    .dataframe th,
-    .dataframe td,
-    .summary-table th,
-    .summary-table td {
-        padding: 12px 15px;
-        border: 1px solid #CCCCCC;
-        text-align: left;
-    }
-    .dataframe thead th,
-    .summary-table th {
-        background-color: #333333;
-        color: #FFFFFF;
-    }
-    .dataframe tbody tr:nth-child(even),
-    .summary-table tr:nth-child(even) {
-        background-color: #F0F0F0;
-    }
-    .dataframe tbody tr:nth-child(odd),
-    .summary-table tr:nth-child(odd) {
-        background-color: #E0E0E0;
-    }
-    .dataframe tbody td,
-    .summary-table td {
-        color: #333333;
-    }
-    .dataframe tbody tr:hover,
-    .summary-table tr:hover {
-        background-color: #D1D1D1;
-    }
-    .table-container {
-        overflow-x: auto;
-    }
-    </style>
-    """, unsafe_allow_html=True)