leaderboard, tree, time slider, need to clean up

app.py

@@ -1,5 +1,6 @@
 from dash import Dash, html, dcc, Input, Output
 import pandas as pd
+import dash_mantine_components as dmc
 from graphs.model_market_share import create_stacked_area_chart, create_world_map, create_range_slider
 from graphs.leaderboard import create_leaderboard
 from graphs.model_characteristics import create_concentration_chart, create_line_plot
@@ -89,6 +90,26 @@ slider = create_range_slider(
     model_topk_df
 )
 
+time_slider = dmc.RangeSlider(
+    id="time-slider",
+    min=model_topk_df['time'].min().timestamp(),
+    max=model_topk_df['time'].max().timestamp(),
+    value=[
+        model_topk_df['time'].min().timestamp(),
+        model_topk_df['time'].max().timestamp()
+    ],
+    step=24 * 60 * 60,
+    color="blue",
+    size="md",
+    radius="xl",
+    marks=[
+        {"value": model_topk_df['time'].min().timestamp(), "label": model_topk_df['time'].min().strftime("%b %Y")},
+        {"value": model_topk_df['time'].max().timestamp(), "label": model_topk_df['time'].max().strftime("%b %Y")}
+    ],
+    style={"width": "70%", "margin": "0 auto"},
+    labelAlwaysOn=False
+)
+
 # Model Characteristics Tab
 language_concentration_area = create_concentration_chart(
     language_concentration_df, 'time', 'metric', 'value', LANG_SEGMENT_ORDER, PALETTE_0
@@ -111,7 +132,11 @@ tree_map = generate_model_treemap(
 )
 
 # App layout
-app.layout = html.Div(
+app.layout = dmc.MantineProvider(
+    theme={"colorScheme": "light",
+           "primaryColor": "blue",
+           "fontFamily": "Inter, sans-serif"},
+    children=[html.Div(
     [
         html.Div(
             [
@@ -128,8 +153,8 @@ app.layout = html.Div(
                                 'padding': '4px 14px',
                                 'fontSize': 13,
                                 'color': 'white',
-                                'backgroundColor': '#2563eb',
                                 'border': 'none',
+                                'backgroundColor': '#228BE6',
                                 'borderRadius': '18px',
                                 'textDecoration': 'none',
                                 'fontWeight': 'bold',
@@ -154,7 +179,7 @@ app.layout = html.Div(
                     dcc.Tab(label='Model Market Share', children=[
                         html.Div([
                             html.Div(children='Select time range to update all graphs below:', style={'fontSize': 16, 'marginBottom': 6, 'marginTop': 20}),
-                            dcc.Graph(figure=slider, id='time-slider', style={'height': '100px'}),
+                            time_slider,
                             html.Div(
                                 id='output-container-range-slider',
                                 style={
@@ -210,7 +235,7 @@ app.layout = html.Div(
         )
     ],
     style={'fontFamily': 'Inter', 'backgroundColor': '#f7f7fa', 'minHeight': '100vh'}
-)
+)])
 
 # Callbacks for interactivity
 
@@ -218,40 +243,32 @@ app.layout = html.Div(
 # On slider change, update output text
 @app.callback(
     Output('output-container-range-slider', 'children'),
-    [Input('time-slider', 'relayoutData')]
+    [Input('time-slider', 'value')]
 )
-def update_output(relayout_data):
-    def format_date(date_str):
-        date = pd.to_datetime(date_str)
-        return date.strftime('%b {S}, %Y').replace('{S}', str(date.day) + (
-            'th' if 11 <= date.day <= 13 else {1: 'st', 2: 'nd', 3: 'rd'}.get(date.day % 10, 'th')
-        ))
-
-    if relayout_data and 'xaxis.range[0]' in relayout_data and 'xaxis.range[1]' in relayout_data:
-        start_time = format_date(relayout_data['xaxis.range[0]'])
-        end_time = format_date(relayout_data['xaxis.range[1]'])
-    else:
-        # Earliest and latest dates in the dataset
-        start_time = format_date(model_topk_df['time'].min())
-        end_time = format_date(model_topk_df['time'].max())
-    return f'{start_time} to {end_time}'
+def update_output(value):
+    if value and len(value) == 2:
+        start_time = pd.to_datetime(value[0], unit='s').strftime("%b %d, %Y")
+        end_time = pd.to_datetime(value[1], unit='s').strftime("%b %d, %Y")
+        return f"Selected time range: {start_time} to {end_time}"
+    return "Select a time range"
 
 # On slider change, update world map
 @app.callback(
     Output('world-map-with-slider', 'figure'),
-    [Input('time-slider', 'relayoutData')]
+    Input('time-slider', 'value')
 )
-def update_map(relayout_data):
-    if relayout_data and 'xaxis.range[0]' in relayout_data and 'xaxis.range[1]' in relayout_data:
-        start_time = pd.to_datetime(relayout_data['xaxis.range[0]']).strftime('%Y-%m-%d')
-        end_time = pd.to_datetime(relayout_data['xaxis.range[1]']).strftime('%Y-%m-%d')
+def update_world_map(value):
+    if value and len(value) == 2:
+        start_time = pd.to_datetime(value[0], unit='s').strftime('%Y-%m-%d')
+        end_time = pd.to_datetime(value[1], unit='s').strftime('%Y-%m-%d')
         updated_fig = create_world_map(
-            country_concentration_df, "time", "metric", "value", start_time=start_time, end_time=end_time
+            country_concentration_df, "time", "metric", "value",
+            start_time=start_time, end_time=end_time
         )
         updated_fig.update_layout(font_family="Inter")
         return updated_fig
-    else:
-        return world_map
+    return world_map
+
 
 # On slider change, update leaderboard
 # @app.callback(
@@ -273,20 +290,21 @@ def update_map(relayout_data):
 # On slider change, update stacked area chart
 @app.callback(
     Output('stacked-area-chart', 'figure'),
-    [Input('time-slider', 'relayoutData')]
+    Input('time-slider', 'value')
 )
-def update_stacked_area(relayout_data):
-    if relayout_data and 'xaxis.range[0]' in relayout_data and 'xaxis.range[1]' in relayout_data:
-        start_time = pd.to_datetime(relayout_data['xaxis.range[0]']).strftime('%Y-%m-%d')
-        end_time = pd.to_datetime(relayout_data['xaxis.range[1]']).strftime('%Y-%m-%d')
+def update_stacked_area(value):
+    if value and len(value) == 2:
+        start_time = pd.to_datetime(value[0], unit='s').strftime('%Y-%m-%d')
+        end_time = pd.to_datetime(value[1], unit='s').strftime('%Y-%m-%d')
         updated_fig = create_stacked_area_chart(
-            model_topk_df, model_gini_df, model_hhi_df, TEMP_MODEL_EVENTS, PALETTE_0,
+            model_topk_df, model_gini_df, model_hhi_df,
+            TEMP_MODEL_EVENTS, PALETTE_0,
             start_time=start_time, end_time=end_time
         )
         updated_fig.update_layout(font_family="Inter")
         return updated_fig
-    else:
-        return model_market_share_area
+    return model_market_share_area
+
     
 # Model Characteristics Tab
 # On dropdown change, update graph

graphs/leaderboard.py

@@ -1,5 +1,6 @@
 import pandas as pd
-from dash import html
+from dash import html, dcc
+import base64
 
 def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_time=None, end_time=None, top_n=10):
     country_icon_map = {
@@ -33,21 +34,24 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
     for df in [country_df, developer_df, model_df]:
         df["time"] = pd.to_datetime(df["time"])
 
+    # change any value that does not equal "org" to "user"
+    filtered_df["org_or_user"] = filtered_df["org_or_user"].where(filtered_df["org_or_user"] == "org", "user")
+
     # Merge country info for developers/models
     developer_df = developer_df.merge(
-        filtered_df[["country", "author", "org_or_user", "model", "downloads"]].drop_duplicates(subset=["author"]),
+        filtered_df[["country", "author", "org_or_user", "model", "downloads", "estimated_parameters"]].drop_duplicates(subset=["author"]),
         left_on="metric", right_on="author", how="left"
     ).drop(columns=["metric"])
 
     model_df = model_df.merge(
-        filtered_df[["country", "author", "downloads", "org_or_user", "model", "merged_modality"]].drop_duplicates(subset=["model"]),
+        filtered_df[["country", "author", "downloads", "org_or_user", "model", "merged_modality", "estimated_parameters"]].drop_duplicates(subset=["model"]),
         left_on="metric", right_on="model", how="left"
     ).drop(columns=["metric"])
 
     # Rename metric columns
     # country_df = country_df.rename(columns={"metric": "country"})
     country_df = country_df.merge(
-        filtered_df[["country", "downloads"]].drop_duplicates(subset=["country"]),
+        filtered_df[["country", "downloads", "estimated_parameters"]].drop_duplicates(subset=["country"]),
         left_on="metric", right_on="country", how="left"
     ).drop(columns=["metric"])
 
@@ -74,17 +78,25 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
         total_value = top["Total Value"].sum()
         top["% of total"] = top["Total Value"] / total_value * 100 if total_value else 0
 
+        # Create a downloadable version of the leaderboard
+        download_top = top.copy()
+        download_top["Total Value"] = download_top["Total Value"].astype(int)
+        download_top["% of total"] = download_top["% of total"].round(2)
+
         # All relevant metadata columns
-        meta_cols = ["country", "author", "downloads", "org_or_user", "merged_modality"]
+        meta_cols = ["country", "author", "downloads", "org_or_user", "merged_modality", "estimated_parameters"]
         # Collect all metadata per top n for each category (country, author, model)
         meta_map = {}
+        download_map = {}
         for name in top["Name"]:
             name_data = df[df[group_col] == name]
             meta_map[name] = {}
+            download_map[name] = {}
             for col in meta_cols:
                 if col in name_data.columns:
                     unique_vals = name_data[col].unique()
                     meta_map[name][col] = list(unique_vals)
+                    download_map[name][col] = list(unique_vals)
         
         # Function to build metadata chips
         def build_metadata(nm):
@@ -111,17 +123,49 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
             # Modality
             for m in meta.get("merged_modality", []):
                 chips.append(("", m))
+        
+            # Estimated Parameters
+            for p in meta.get("estimated_parameters", []):
+                if pd.notna(p):  # Check if p is not NaN
+                    if p >= 1e9:
+                        p_str = f"{p/1e9:.1f}B"
+                    elif p >= 1e6:
+                        p_str = f"{p/1e6:.1f}M"
+                    elif p >= 1e3:
+                        p_str = f"{p/1e3:.1f}K"
+                    else:
+                        p_str = str(p)
+                    chips.append(("⚙️", p_str))
             return chips
         
+        # Function to create downloadable dataframe
+        def build_download_metadata(nm):
+            meta = download_map.get(nm, {})
+            download_info = {}
+            for col in meta_cols:
+                # don't add empty columns
+                if col not in meta or not meta[col]:
+                    continue
+                vals = meta.get(col, [])
+                if vals:
+                    # Join list into a single string for CSV
+                    download_info[col] = ", ".join(str(v) for v in vals)
+                else:
+                    download_info[col] = ""
+            return download_info
+        
         # Apply metadata builder to top dataframe
         top["Metadata"] = top["Name"].map(build_metadata)
+        download_info_list = [build_download_metadata(nm) for nm in download_top["Name"]]
+        download_info_df = pd.DataFrame(download_info_list)
+        download_top = pd.concat([download_top, download_info_df], axis=1)
 
-        return top[["Name", "Metadata", "% of total"]]
+        return top[["Name", "Metadata", "% of total"]], download_top
 
     # Build leaderboards
-    top_countries = get_top_n_leaderboard(country_df, "country", top_n)
-    top_developers = get_top_n_leaderboard(developer_df, "author", top_n)
-    top_models = get_top_n_leaderboard(model_df, "model", top_n)
+    top_countries, download_top_countries = get_top_n_leaderboard(country_df, "country", top_n)
+    top_developers, download_top_developers = get_top_n_leaderboard(developer_df, "author", top_n)
+    top_models, download_top_models = get_top_n_leaderboard(model_df, "model", top_n)
 
     # Chip renderer
     def chip(text, bg_color="#F0F0F0"):
@@ -202,9 +246,37 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
                 )
             ]
         )
+    
+    # Helper to convert DataFrame to CSV and encode for download
+    def df_to_download_link(df, filename):
+        csv_string = df.to_csv(index=False)
+        b64 = base64.b64encode(csv_string.encode()).decode()
+        return html.Div(
+            html.A(
+            "Download CSV",
+            id=f"download-{filename}",
+            download=f"{filename}.csv",
+            href=f"data:text/csv;base64,{b64}",
+            target="_blank",
+            style={
+                "display": "inline-block",
+                "marginBottom": "10px",
+                "marginRight": "15px",
+                "marginTop": "30px",
+                "padding": "6px 16px",
+                "backgroundColor": "#2196F3",
+                "color": "white",
+                "borderRadius": "6px",
+                "textDecoration": "none",
+                "fontWeight": "bold",
+                "fontSize": "14px"
+            }
+            ),
+            style={"textAlign": "right"}
+        )
 
     # Table renderer
-    def render_table(df, title, chip_color="#F0F0F0", bar_color="#4CAF50"):
+    def render_table(df, download_df, title, chip_color="#F0F0F0", bar_color="#4CAF50", filename="data"):
         return html.Div([
             html.H4(title, style={"textAlign": "left", "marginBottom": "10px", "fontSize": "20px"}),
             html.Table([
@@ -222,14 +294,15 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
                         html.Td(progress_bar(row["% of total"], bar_color), style={"textAlign": "center"})
                     ]) for idx, row in df.iterrows()
                 ])
-            ], style={"borderCollapse": "collapse", "width": "100%"})
+            ], style={"borderCollapse": "collapse", "width": "100%"}),
+            df_to_download_link(download_df, filename),
         ], style={"marginBottom": "20px"})
 
     # Layout with 3 stacked tables
     layout = html.Div([
-        render_table(top_countries, "Top Countries", chip_color="#FCE8E6", bar_color="#FF6F61"),
-        render_table(top_developers, "Top Developers", chip_color="#E6F4EA", bar_color="#4CAF50"),
-        render_table(top_models, "Top Models", chip_color="#E8F0FE", bar_color="#2196F3"),
+        render_table(top_countries, download_top_countries, "Top Countries", chip_color="#FCE8E6", bar_color="#FF6F61", filename="top_countries"),
+        render_table(top_developers, download_top_developers, "Top Developers", chip_color="#E6F4EA", bar_color="#4CAF50", filename="top_developers"),
+        render_table(top_models, download_top_models, "Top Models", chip_color="#E8F0FE", bar_color="#2196F3", filename="top_models"),
     ])
 
     return layout

graphs/model_market_share.py

@@ -285,54 +285,6 @@ def create_world_map(
         row=1,
         col=1,
     )
-
-    # Country center coordinates for labels
-    # country_centers = {
-    #     "USA": {"lat": 39.8, "lon": -98.5},
-    #     "CHN": {"lat": 35.8, "lon": 104.2},
-    #     "DEU": {"lat": 51.2, "lon": 10.4},
-    #     "GBR": {"lat": 55.4, "lon": -3.4},
-    #     "FRA": {"lat": 46.6, "lon": 2.2},
-    #     "JPN": {"lat": 36.2, "lon": 138.3},
-    #     "IND": {"lat": 20.6, "lon": 78.9},
-    #     "CAN": {"lat": 56.1, "lon": -106.3},
-    #     "RUS": {"lat": 61.5, "lon": 105.3},
-    #     "BRA": {"lat": -14.2, "lon": -51.9},
-    #     "AUS": {"lat": -25.3, "lon": 133.8},
-    #     "KOR": {"lat": 35.9, "lon": 127.8},
-    # }
-
-    # # Add initial labels using scattergeo instead of annotations
-    # label_lons = []
-    # label_lats = []
-    # label_texts = []
-
-    # for _, country in top_countries.iterrows():
-    #     country_code = country["country_code"]
-    #     if country_code in country_centers:
-    #         center = country_centers[country_code]
-    #         label_lons.append(center["lon"])
-    #         label_lats.append(center["lat"])
-    #         label_texts.append(f"{country['percentage']:.1f}%")
-
-    # # Add text labels as a scattergeo trace
-    # fig.add_trace(
-    #     go.Scattergeo(
-    #         lon=label_lons,
-    #         lat=label_lats,
-    #         text=label_texts,
-    #         mode="text",
-    #         textfont=dict(
-    #             color="#ffffff", size=13, family="Inter, system-ui, sans-serif"
-    #         ),
-    #         textposition="middle center",
-    #         showlegend=False,
-    #         hoverinfo="skip",
-    #         geo="geo",
-    #     ),
-    #     row=1,
-    #     col=1,
-    # )
     
     # Update layout
     fig.update_layout(

graphs/tree.py

@@ -1,8 +1,29 @@
 import plotly.express as px
 import pandas as pd
 
+PALETTE_0 = [
+    "#335C67",
+    "#FFF3B0",
+    "#E09F3E",
+    "#9E2A2B",
+    "#540B0E"
+]
+
 def generate_model_treemap(df, parent_col='merged_derived_from', child_col='model', value_col='downloads'):
-    df[parent_col] = str(df[parent_col][0])
+    # filtered_df[parent_col] = filtered_df[parent_col].apply(lambda x: str(x[0]) if isinstance(x, list) and x else None)
+    
+    df = pd.read_pickle('data_frames/filtered_tree_df.pkl')
+    # Filter out nan, No parent, and Unsure
+    df = df[~df[parent_col].isin([None, "['Unsure']", 'nan'])]
+
+    # Find all models that act as a parent
+    parent_models = set(df[parent_col].dropna())
+
+    # Assign empty parent only if row has no parent and is not itself a parent
+    df[parent_col] = df[parent_col].where(
+        df[parent_col].notna() | df[child_col].isin(parent_models),
+        other=""
+    )
 
     fig = px.treemap(
         df,
@@ -12,4 +33,110 @@ def generate_model_treemap(df, parent_col='merged_derived_from', child_col='mode
         color=value_col,
         color_continuous_scale='Viridis'
     )
+
+    fig.update_layout(
+        height=1200,  # make the plot tall
+        margin=dict(t=50, l=25, r=25, b=25)  # add some breathing room
+    )
+    
     return fig
+
+# def generate_model_treemap(df, parent_col='merged_derived_from', child_col='model', value_col='downloads'):
+#     # iterate over the rows and stringify the lists in 'merged_derived_from'
+
+#     df.to_pickle('filtered_tree_df.pkl')
+
+#     fig = px.icicle(
+#         df,
+#         path=[parent_col, child_col],
+#         values=value_col,
+#         hover_data=['author', 'estimated_parameters', 'created'],
+#         color=value_col,
+#         color_continuous_scale='Viridis'
+#     )
+
+#     fig.update_layout(
+#         height=1400,
+#         margin=dict(t=50, l=25, r=25, b=25)
+#     )
+#     return fig
+
+
+# import plotly.graph_objects as go
+# import networkx as nx
+# import pandas as pd
+
+# def generate_model_treemap(df, parent_col='merged_derived_from', child_col='model',
+#                         value_col='downloads', top_n=1000):
+
+#     # Fill missing parents
+#     df[parent_col] = str(df[parent_col][0])
+
+#     # Keep only top_n by downloads
+#     df = df.sort_values(value_col, ascending=False).head(top_n)
+
+#     # Build directed graph
+#     G = nx.DiGraph()
+#     for _, row in df.iterrows():
+#         parent = row[parent_col]
+#         child = row[child_col]
+#         G.add_edge(parent, child, weight=row.get(value_col, 1))
+
+#     # Layout positions (smaller k → tighter graph)
+#     pos = nx.spring_layout(G, k=0.3, seed=42)
+
+#     # Edges
+#     edge_x, edge_y = [], []
+#     for parent, child in G.edges():
+#         x0, y0 = pos[parent]
+#         x1, y1 = pos[child]
+#         edge_x += [x0, x1, None]
+#         edge_y += [y0, y1, None]
+
+#     edge_trace = go.Scatter(
+#         x=edge_x, y=edge_y,
+#         line=dict(width=0.8, color="#888"),
+#         hoverinfo="none",
+#         mode="lines"
+#     )
+
+#     # Nodes
+#     node_x, node_y, sizes, texts = [], [], [], []
+#     for node in G.nodes():
+#         x, y = pos[node]
+#         node_x.append(x)
+#         node_y.append(y)
+#         downloads = df.loc[df[child_col] == node, value_col].sum()
+#         sizes.append(max(10, downloads**0.3))
+#         texts.append(f"{node}<br>Downloads: {downloads}")
+
+#     node_trace = go.Scatter(
+#         x=node_x, y=node_y,
+#         mode="markers+text",
+#         text=[n for n in G.nodes()],
+#         textposition="top center",
+#         hovertext=texts,
+#         hoverinfo="text",
+#         marker=dict(
+#             showscale=True,
+#             colorscale="Viridis",
+#             color=sizes,
+#             size=sizes,
+#             colorbar=dict(
+#                 thickness=15,
+#                 title=f"{value_col} (scaled)",
+#                 xanchor="left",
+#             ),
+#             line_width=2
+#         )
+#     )
+
+#     return go.Figure(data=[edge_trace, node_trace],
+#                      layout=go.Layout(
+#                          title=f"Model Tree (Top {top_n} by {value_col})",
+#                          showlegend=False,
+#                          hovermode="closest",
+#                          margin=dict(b=20, l=5, r=5, t=40),
+#                          xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
+#                          yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
+#                      ))

requirements.txt

@@ -1,4 +1,5 @@
 pandas
 dash
 plotly
-gunicorn
+gunicorn
+dash-mantine-components