consolidate queries into function

app.py

@@ -3,7 +3,12 @@ import pandas as pd
 import dash_mantine_components as dmc
 import duckdb
 import time
-from graphs.leaderboard import button_style, get_top_n_leaderboard, render_table_content
+from graphs.leaderboard import (
+    button_style,
+    get_top_n_leaderboard,
+    render_table_content,
+    build_leaderboard_query,
+)
 from dash_iconify import DashIconify
 
 # Initialize the app
@@ -796,138 +801,8 @@ def _get_filtered_top_n_from_duckdb(
         start_str = str(start)
         end_str = str(end)
 
-        # If grouping by country, transform some country values
-        if group_col == "org_country_single":
-            group_expr = """CASE 
-                WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                ELSE org_country_single
-            END"""
-        else:
-            group_expr = group_col
-
-        # Derived-author requires author->country lookup; build separate SQL for that case
-        if group_col == "derived_author":
-            query = f"""
-            WITH base_data AS (
-                SELECT
-                    {group_expr} AS group_key,
-                    CASE 
-                        WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                        WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                        ELSE org_country_single
-                    END AS org_country_single,
-                    author,
-                    derived_author,
-                    merged_country_groups_single,
-                    merged_modality,
-                    model,
-                    time,
-                    downloadsAllTime
-                FROM {view}
-            ),
-
-            author_country_lookup AS (
-                SELECT DISTINCT
-                    author,
-                    FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
-                FROM base_data
-                WHERE author IS NOT NULL
-            ),
-
-            model_metrics AS (
-                SELECT
-                    model,
-                    group_key,
-                    ANY_VALUE(org_country_single) AS org_country_single,
-                    ANY_VALUE(author) AS author,
-                    ANY_VALUE(derived_author) AS derived_author,
-                    ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
-                    ANY_VALUE(merged_modality) AS merged_modality,
-                    COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
-                    - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
-                    AS total_downloads
-                FROM base_data
-                GROUP BY model, group_key
-            ),
-
-            total_downloads_cte AS (
-                SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
-            )
-
-            SELECT
-                mm.model,
-                mm.group_key,
-                COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
-                mm.author,
-                mm.derived_author,
-                mm.merged_country_groups_single,
-                mm.merged_modality,
-                mm.total_downloads,
-                CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
-            FROM model_metrics mm
-            LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
-            CROSS JOIN total_downloads_cte td
-            WHERE mm.total_downloads > 0
-            ORDER BY mm.total_downloads DESC
-            LIMIT {top_n * 10};
-            """
-        else:
-            query = f"""
-            WITH base_data AS (
-                SELECT
-                    {group_expr} AS group_key,
-                    CASE 
-                        WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                        WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                        ELSE org_country_single
-                    END AS org_country_single,
-                    author,
-                    derived_author,
-                    merged_country_groups_single,
-                    merged_modality,
-                    model,
-                    time,
-                    downloadsAllTime
-                FROM {view}
-            ),
-
-            model_metrics AS (
-                SELECT
-                    model,
-                    group_key,
-                    ANY_VALUE(org_country_single) AS org_country_single,
-                    ANY_VALUE(author) AS author,
-                    ANY_VALUE(derived_author) AS derived_author,
-                    ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
-                    ANY_VALUE(merged_modality) AS merged_modality,
-                    COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
-                    - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
-                    AS total_downloads
-                FROM base_data
-                GROUP BY model, group_key
-            ),
-
-            total_downloads_cte AS (
-                SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
-            )
-
-            SELECT
-                mm.model,
-                mm.group_key,
-                mm.org_country_single,
-                mm.author,
-                mm.derived_author,
-                mm.merged_country_groups_single,
-                mm.merged_modality,
-                mm.total_downloads,
-                CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
-            FROM model_metrics mm
-            CROSS JOIN total_downloads_cte td
-            WHERE mm.total_downloads > 0
-            ORDER BY mm.total_downloads DESC
-            LIMIT {top_n * 10};
-            """
+        # Build query using shared function
+        query = build_leaderboard_query(group_col, top_n, start_str, end_str, view=view)
 
         # execute using the fresh local connection
         result_df = local_con.execute(query).fetchdf()

graphs/leaderboard.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from dash import html, dcc
+from dash import html
 from dash_iconify import DashIconify
 import dash_mantine_components as dmc
 import base64
@@ -496,28 +496,12 @@ def create_fresh_duckdb_with_views():
     return local_con
 
 
-def get_top_n_from_duckdb(
-    con, group_col, top_n=10, time_filter=None, view="all_downloads"
-):
+def build_leaderboard_query(group_col, top_n, start_str, end_str, view="all_downloads"):
     """
-    Query DuckDB directly to get model-level rows with per-model total_downloads (delta or full)
-    Returns rows similar to _get_filtered_top_n_from_duckdb in app.py.
-    NOTE: This function now opens a fresh DuckDB connection internally and ignores
-    any external connection passed in. Keep signature for compatibility.
+    Build and return the SQL query string for the given grouping.
+    Encapsulates common parts used both in app.py and in this module.
     """
-    # Compute date window
-    if time_filter and len(time_filter) == 2:
-        start = pd.to_datetime(time_filter[0], unit="s")
-        end = pd.to_datetime(time_filter[1], unit="s")
-    else:
-        start = pd.to_datetime("1970-01-01")
-        # We cannot access end_dt here; rely on time_filter for end in typical use.
-        end = pd.Timestamp.now()
-
-    start_str = str(start)
-    end_str = str(end)
-
-    # If grouping by country, transform some country values
+    # handle country grouping normalization
     if group_col == "org_country_single":
         group_expr = """CASE 
             WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
@@ -527,9 +511,9 @@ def get_top_n_from_duckdb(
     else:
         group_expr = group_col
 
-    # Derived author special-case
+    # Derived-author special-case (uses author-derived lookups)
     if group_col == "derived_author":
-        query = f"""
+        return f"""
         WITH base_data AS (
             SELECT
                 {group_expr} AS group_key,
@@ -550,67 +534,18 @@ def get_top_n_from_duckdb(
 
         author_country_lookup AS (
             SELECT DISTINCT
-                author,
-                FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
-            FROM base_data
-            WHERE author IS NOT NULL
-        ),
-
-        model_metrics AS (
-            SELECT
-                model,
-                group_key,
-                ANY_VALUE(org_country_single) AS org_country_single,
-                ANY_VALUE(author) AS author,
-                ANY_VALUE(derived_author) AS derived_author,
-                ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
-                ANY_VALUE(merged_modality) AS merged_modality,
-                COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
-                - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
-                AS total_downloads
+                derived_author,
+                FIRST_VALUE(org_country_single) OVER (PARTITION BY derived_author ORDER BY downloadsAllTime DESC) AS derived_author_country
             FROM base_data
-            GROUP BY model, group_key
+            WHERE derived_author IS NOT NULL
         ),
 
-        total_downloads_cte AS (
-            SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
-        )
-
-        SELECT
-            mm.model,
-            mm.group_key,
-            COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
-            mm.author,
-            mm.derived_author,
-            mm.merged_country_groups_single,
-            mm.merged_modality,
-            mm.total_downloads,
-            CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
-        FROM model_metrics mm
-        LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
-        CROSS JOIN total_downloads_cte td
-        WHERE mm.total_downloads > 0
-        ORDER BY mm.total_downloads DESC
-        LIMIT {top_n * 10};
-        """
-    else:
-        query = f"""
-        WITH base_data AS (
-            SELECT
-                {group_expr} AS group_key,
-                CASE 
-                    WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                    WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                    ELSE org_country_single
-                END AS org_country_single,
-                author,
+        author_merged_country_lookup AS (
+            SELECT DISTINCT
                 derived_author,
-                merged_country_groups_single,
-                merged_modality,
-                model,
-                time,
-                downloadsAllTime
-            FROM {view}
+                FIRST_VALUE(merged_country_groups_single) OVER (PARTITION BY derived_author ORDER BY downloadsAllTime DESC) AS derived_author_merged_country
+            FROM base_data
+            WHERE derived_author IS NOT NULL
         ),
 
         model_metrics AS (
@@ -636,19 +571,102 @@ def get_top_n_from_duckdb(
         SELECT
             mm.model,
             mm.group_key,
-            mm.org_country_single,
+            acl.derived_author_country AS org_country_single,
+            amc.derived_author_merged_country AS merged_country_groups_single,
             mm.author,
             mm.derived_author,
-            mm.merged_country_groups_single,
             mm.merged_modality,
             mm.total_downloads,
             CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
         FROM model_metrics mm
+        LEFT JOIN author_country_lookup acl ON mm.group_key = acl.derived_author
+        LEFT JOIN author_merged_country_lookup amc ON mm.group_key = amc.derived_author
         CROSS JOIN total_downloads_cte td
         WHERE mm.total_downloads > 0
         ORDER BY mm.total_downloads DESC
         LIMIT {top_n * 10};
         """
+    # Generic grouping SQL
+    return f"""
+    WITH base_data AS (
+        SELECT
+            {group_expr} AS group_key,
+            CASE 
+                WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
+                WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
+                ELSE org_country_single
+            END AS org_country_single,
+            author,
+            derived_author,
+            merged_country_groups_single,
+            merged_modality,
+            model,
+            time,
+            downloadsAllTime
+        FROM {view}
+    ),
+
+    model_metrics AS (
+        SELECT
+            model,
+            group_key,
+            ANY_VALUE(org_country_single) AS org_country_single,
+            ANY_VALUE(author) AS author,
+            ANY_VALUE(derived_author) AS derived_author,
+            ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
+            ANY_VALUE(merged_modality) AS merged_modality,
+            COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
+            - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
+            AS total_downloads
+        FROM base_data
+        GROUP BY model, group_key
+    ),
+
+    total_downloads_cte AS (
+        SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
+    )
+
+    SELECT
+        mm.model,
+        mm.group_key,
+        mm.org_country_single,
+        mm.author,
+        mm.derived_author,
+        mm.merged_country_groups_single,
+        mm.merged_modality,
+        mm.total_downloads,
+        CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
+    FROM model_metrics mm
+    CROSS JOIN total_downloads_cte td
+    WHERE mm.total_downloads > 0
+    ORDER BY mm.total_downloads DESC
+    LIMIT {top_n * 10};
+    """
+
+
+def get_top_n_from_duckdb(
+    con, group_col, top_n=10, time_filter=None, view="all_downloads"
+):
+    """
+    Query DuckDB directly to get model-level rows with per-model total_downloads (delta or full)
+    Returns rows similar to _get_filtered_top_n_from_duckdb in app.py.
+    NOTE: This function now opens a fresh DuckDB connection internally and ignores
+    any external connection passed in. Keep signature for compatibility.
+    """
+    # Compute date window
+    if time_filter and len(time_filter) == 2:
+        start = pd.to_datetime(time_filter[0], unit="s")
+        end = pd.to_datetime(time_filter[1], unit="s")
+    else:
+        start = pd.to_datetime("1970-01-01")
+        # We cannot access end_dt here; rely on time_filter for end in typical use.
+        end = pd.Timestamp.now()
+
+    start_str = str(start)
+    end_str = str(end)
+
+    # Build SQL using the shared helper
+    query = build_leaderboard_query(group_col, top_n, start_str, end_str, view=view)
 
     # Open a fresh in-memory connection that creates the views, run the query, close.
     conn_local = create_fresh_duckdb_with_views()