Dashnoard test

app.py

@@ -4,7 +4,9 @@ import pandas as pd
 import os
 import time
 import datetime
-from utils import fetch_runs_to_df, fetch_run, fetch_models_to_df
+from utils import fetch_competition_summary, fetch_models_evaluation
+
+st.set_page_config(layout="wide")
 
 ### WANDB
 
@@ -24,11 +26,12 @@ api = wandb.Api()
 
 # Define available projects (bookmarks)
 projects = {
-    "Competition 1": {"entity": "urbaniak-bruno-safescanai", "project": "pytorch-intro", "description": "This is a beginner-friendly competition using PyTorch."},
-    "Competition 2": {"entity": "urbaniak-bruno-safescanai", "project": "basic-intro", "description": "This competition focuses on basic introduction to AI models."},
-    "Competition 3 (mamymodelexd)": {"entity": "urbaniak-bruno-safescanai", "project": "simple-cnn", "description": "This is an advanced competition for CNN model training."},
-    "Competition 4": {"entity": "urbaniak-bruno-safescanai", "project": "model-validation", "description": "This competition covers techniques for model validation."},
-    "Competition 5": {"entity": "safe-scan-ai", "project": "melanoma-1", "description": "This competition focuses on melanoma detection."},
+    "Melanoma TestNet": {
+        "entity": "safe-scan-ai",
+        "project": "melanoma-testnet",
+        "description": "This is a testnet competition for melanoma detection."
+    },
+    
     # Add more projects as needed
 }
 
@@ -65,68 +68,54 @@ def update_leader_info(leader_info, competition, best_model):
             leader_info[competition]["UID"] = best_model["ID"]
             leader_info[competition]["Days on Top"] = 1
     return leader_info[competition]
-
-
 @st.cache_data()
 def load_competition_data(last_update_time=None):
-    ranking_dfs = {}
-    for competition, details in projects.items():
-        df = fetch_runs_to_df(api, projects, competition)
-        if not df.empty:
-            # Convert to DataFrame
-            ranking_df = fetch_models_to_df(api, projects, competition, df)
-
-            # Rank the models by accuracy, then recall, then ROC AUC
-            rank_by = ["Recall", "Accuracy", "ROC AUC"]
-            ascending = [False, False, False]
-            ranking_df = ranking_df.sort_values(by=rank_by, ascending=ascending)
-        
-            ranking_dfs[competition] = ranking_df
+    competition_summaries = {}
+    model_evaluations = {}
+    for competition in projects:
+        competition_summaries[competition] = fetch_competition_summary(api, projects, competition)
+        model_evaluations[competition] = fetch_models_evaluation(api, projects, competition)
     
-    # Update the timestamp of the last update
     last_update_time = time.time()
-    return ranking_dfs, last_update_time
-
+    return competition_summaries, model_evaluations, last_update_time
 
 # Streamlit app main function
 def main():
-    # Set Streamlit page configuration to wide
-    st.set_page_config(layout="wide")
     st.title("LaUltimate Dashboard")
+    st.title("JEDZIESZMALINA")
     st.subheader("Welcome to the Competition Dashboard!")
     st.write("Explore the various AI competitions and their respective rankings. Select a competition to view more details and rankings.")
 
-    # Define the update interval (e.g., 10 minutes)
-    update_interval = 20 * 60  # 10 minutes in seconds
+    update_interval = 20 * 60  # 20 minutes in seconds
 
-    # Initialize session state to track the last update time
     if 'last_update_time' not in st.session_state:
         st.session_state.last_update_time = None
-    # Initialize or update leader_info
     if "leader_info" not in st.session_state:
         st.session_state.leader_info = {}
 
-    # Check if we need to refresh the data
     if st.session_state.last_update_time is None or (time.time() - st.session_state.last_update_time > update_interval):
-        ranking_dfs, st.session_state.last_update_time = load_competition_data(st.session_state.last_update_time)
-
-        # Update leader info for each competition
-        for competition in ranking_dfs:
-            best_model = ranking_dfs[competition].iloc[0]
-            st.session_state.leader_info[competition] = update_leader_info(st.session_state.leader_info, competition, best_model)
-            
+        competition_summaries, model_evaluations, st.session_state.last_update_time = load_competition_data(st.session_state.last_update_time)
+
+        for competition in projects:
+            if not model_evaluations[competition].empty:
+                best_model = model_evaluations[competition].iloc[0]
+                st.session_state.leader_info[competition] = update_leader_info(st.session_state.leader_info, competition, best_model)
+            else:
+                st.session_state.leader_info[competition] = {
+                    "Username": "N/A", "Model Name": "N/A", "Hotkey": "N/A",
+                    "Date": "N/A", "UID": "N/A", "Days on Top": "N/A"
+                }
     else:
-        ranking_dfs, _ = load_competition_data(st.session_state.last_update_time)
+        competition_summaries, model_evaluations, _ = load_competition_data(st.session_state.last_update_time)
 
-    # Display the time of the last update
     st.write(f"Last updated: {time.ctime(st.session_state.last_update_time)}")
 
     st.markdown("### Competitions")
 
     st.write("### Select Competition")
 
-    # Create a header for the table with additional columns
-    cols = st.columns([1, 3, 2, 2, 2, 2, 1, 2])  # Adjust column width ratios if needed
+    # Create a header for the table
+    cols = st.columns([1, 3, 2, 2, 2, 2, 1, 2])
     cols[0].write("Index")
     cols[1].write("Competition Name")
     cols[2].write("Leader")
@@ -136,32 +125,23 @@ def main():
     cols[6].write("Hotkey")
     cols[7].write("Days on Top")
 
-
-    # Display the project names in a table-like structure with buttons and placeholders for additional data
     for index, (competition, details) in enumerate(projects.items(), start=1):
-        best_model = ranking_dfs[competition].iloc[0]
-
-        # Use the leader_info from session_state
-        leader_info = st.session_state.get("leader_info")
-
+        leader_info = st.session_state.get("leader_info", {}).get(competition, {})
 
         cols = st.columns([1, 3, 2, 2, 2, 2, 1, 2])
         cols[0].write(index)
         if cols[1].button(competition):
-            st.experimental_set_query_params(comp=competition)
-        cols[2].write(leader_info[competition]["Username"])
-        cols[3].write(leader_info[competition]["Model Name"])
-        cols[4].write(leader_info[competition]["Date"])
-        cols[5].write(leader_info[competition]["UID"])
-        cols[6].write(leader_info[competition]["Hotkey"])
-        cols[7].write(leader_info[competition]["Days on Top"])
-
-
-    # Get query parameters to identify which competition page to show
-    query_params = st.experimental_get_query_params()  
-
+            st.query_params["comp"] = competition
+        cols[2].write(leader_info.get("Username", "N/A"))
+        cols[3].write(leader_info.get("Model Name", "N/A"))
+        cols[4].write(leader_info.get("Date", "N/A"))
+        cols[5].write(leader_info.get("UID", "N/A"))
+        cols[6].write(leader_info.get("Hotkey", "N/A"))
+        cols[7].write(leader_info.get("Days on Top", "N/A"))
+
+    query_params = st.query_params
     if "comp" in query_params:
-        competition_name = query_params["comp"][0]
+        competition_name = query_params["comp"]
         st.write(f"Selected competition: {competition_name}")
         competition_details = projects.get(competition_name, {})
         description = competition_details.get("description", "No description available.")
@@ -169,15 +149,23 @@ def main():
         st.header(f"Competition: {competition_name}")
         st.write(description)
 
-        df = ranking_dfs[competition_name]
-        if not df.empty:
-            st.dataframe(df)
+        st.subheader("Competition Summary")
+        competition_summary_df = competition_summaries.get(competition_name, pd.DataFrame())
+        if not competition_summary_df.empty:
+            st.dataframe(competition_summary_df)
+        else:
+            st.warning("No competition summary data available.")
+
+        st.subheader("Models Evaluation")
+        models_evaluation_df = model_evaluations.get(competition_name, pd.DataFrame())
+        if not models_evaluation_df.empty:
+            st.dataframe(models_evaluation_df)
         else:
-            st.warning("No runs available for ranking.")
+            st.warning("No models evaluation data available.")
     else:
         st.write("Please select a competition to view details.")
 
 
 # Run the app
 if __name__ == "__main__":
-    main()
+    main()

utils.py

@@ -1,109 +1,97 @@
 import streamlit as st
 import pandas as pd
 import wandb
+import datetime
 
-def fetch_runs_to_df(api, projects, selected_project):
-    data = []
 
-    if selected_project == "All":
-        # return all runs from all projects
-        for project_name, details in projects.items():
-            entity = details["entity"]
-            project = details["project"]
-            runs = api.runs(f"{entity}/{project}")
-            for run in runs:
+def fetch_competition_summary(api, projects, selected_project):
+    data = []
+    st.write(f"Fetching competition summary for {selected_project}")
+    
+    entity = projects[selected_project]["entity"]
+    project = projects[selected_project]["project"]
+    
+    runs = api.runs(f"{entity}/{project}")
+    st.write(f"Number of runs to process: {len(runs)}")
+    
+    for run in runs:
+        try:
+            summary = run.summary
+            if summary.get("validator_id") and summary.get("winning_hotkey"):
                 data.append({
-                    "Run Name": run.name,
                     "ID": run.id,
+                    "Validator ID": summary.get("validator_id"),
+                    "Winning Hotkey": summary.get("winning_hotkey"),
+                    "Run Time (s)": summary.get("run_time_s"),
                     "Created At": run.created_at,
-                    "State": run.state,
-                    "Tags": ", ".join(run.tags)  # Join tags into a single string
                 })
-        df = pd.DataFrame(data)
+        except Exception as e:
+            st.write(f"Error processing run {run.id}: {str(e)}")
     
-    else:
-        # Get the selected project's details
-        entity = projects[selected_project]["entity"]
-        project = projects[selected_project]["project"]
-        runs = api.runs(f"{entity}/{project}")
-        for run in runs:
-            data.append({
-                "Run Name": run.name,
-                "ID": run.id,
-                "Created At": run.created_at,
-                "State": run.state,
-                "Tags": ", ".join(run.tags)  # Join tags into a single string
-            })
-        df = pd.DataFrame(data)
-
+    df = pd.DataFrame(data)
+    if not df.empty:
+        df['Created At'] = pd.to_datetime(df['Created At'])
+        df = df.sort_values(by="Created At", ascending=False)
+    
+    st.write(f"Competition summary data: {len(df)} rows")
     return df
 
-def fetch_run(api, projects, selected_project, selected_run_id):
-    # Fetch run details based on the selected project
-    if selected_project == "All":
-        # Find the project for the selected run_id
-        for project_name, details in projects.items():
-            entity = details["entity"]
-            project = details["project"]
-            try:
-                run = api.run(f"{entity}/{project}/{selected_run_id}")
-                break
-            except wandb.errors.CommError:
-                continue
-        else:
-            st.error(f"Run ID {selected_run_id} not found in any project.")
-    else:
-        entity = projects[selected_project]["entity"]
-        project = projects[selected_project]["project"]
-        run = api.run(f"{entity}/{project}/{selected_run_id}")
-
-    return run
-
-
-def fetch_models_to_df(api, projects, selected_project, df):
+def fetch_models_evaluation(api, projects, selected_project):
     data = []
-    for index, row in df.iterrows():
+    st.write(f"Fetching models evaluation for {selected_project}")
+    
+    entity = projects[selected_project]["entity"]
+    project = projects[selected_project]["project"]
+    
+    runs = api.runs(f"{entity}/{project}")
+    st.write(f"Number of runs to process: {len(runs)}")
+    
+    for run in runs:
         try:
-            if selected_project == "All":
-                # Determine the project for the current run
-                for project_name, details in projects.items():
-                    entity = details["entity"]
-                    project = details["project"]
-                    try:
-                        run = api.run(f"{entity}/{project}/{row['ID']}")
-                        break
-                    except wandb.errors.CommError:
-                        continue
-                else:
-                    st.error(f"Run ID {row['ID']} not found in any project.")
-                    continue
-            else:
-                entity = projects[selected_project]["entity"]
-                project = projects[selected_project]["project"]
-                run = api.run(f"{entity}/{project}/{row['ID']}")
-
-            metrics = run.summary
-            model_name = "Unknown"
-            artifacts = run.logged_artifacts()
-            for artifact in artifacts:
-                if artifact.type == "model":  # Assuming the artifact type is "model"
-                    model_name = artifact.name  # Use the artifact's name as the model name
-                    break
-
-            data.append({
-                "Username": run.user.username,
-                "Model Name": metrics.get("model_name"),  # Add model name to the table
-                "Hotkey": metrics.get("hotkey"),  # Add model name to the table
-                # "Run Name": row["Run Name"],
-                "ID": row["ID"],
-                "Number of tested entries": metrics.get("tested_entries"),
-                # "Run Time": metrics.get("model_test_run_time"),
-                "Accuracy": metrics.get("accuracy"),  # Example metric
-                "Recall": metrics.get("recall"),  # Example metric
-                "ROC AUC": metrics.get("roc_auc"),  # Example metric
-            })
-        except wandb.errors.CommError:
-            continue
+            summary = run.summary
+            if summary.get("accuracy") is not None:  # Assuming runs with accuracy are model evaluations
+                data.append({
+                    "ID": run.id,
+                    "Username": run.user.username,
+                    "Model Name": summary.get("model_name", "N/A"),
+                    "Hotkey": summary.get("hotkey", "N/A"),
+                    "Accuracy": summary.get("accuracy"),
+                    "Recall": summary.get("recall"),
+                    "ROC AUC": summary.get("roc_auc"),
+                    "Created At": run.created_at,
+                })
+        except Exception as e:
+            st.write(f"Error processing run {run.id}: {str(e)}")
+    
+    df = pd.DataFrame(data)
+    if not df.empty:
+        df['Created At'] = pd.to_datetime(df['Created At'])
+        df = df.sort_values(by="Created At", ascending=False)
+    
+    st.write(f"Models evaluation data: {len(df)} rows")
+    return df
 
-    data_df = pd.DataFrame(data)
-    return data_df
+def update_leader_info(leader_info, competition, best_model):
+    current_date = datetime.now().strftime("%Y-%m-%d")
+    if leader_info.get(competition) is None:
+        leader_info[competition] = {
+            "Username": best_model["Username"],
+            "Model Name": best_model["Model Name"],
+            "Hotkey": best_model["Hotkey"],
+            "Date": current_date,
+            "UID": best_model["ID"],
+            "Days on Top": 1
+        }
+    else:
+        if leader_info[competition]["UID"] == best_model["ID"]:
+            leader_info[competition]["Days on Top"] += 1
+        else:
+            leader_info[competition] = {
+                "Username": best_model["Username"],
+                "Model Name": best_model["Model Name"],
+                "Hotkey": best_model["Hotkey"],
+                "Date": current_date,
+                "UID": best_model["ID"],
+                "Days on Top": 1
+            }
+    return leader_info[competition]