fix toggle & caching issue

app.py

@@ -11,12 +11,57 @@ app = Dash()
 server = app.server
 
 
+# Add dataset URLs (used by the helper to create views)
+HF_DATASET_ID = "mmpr/open_model_evolution_data"
+hf_parquet_url_1 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/all_downloads_with_annotations.parquet"
+hf_parquet_url_2 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/one_year_rolling.parquet"
+
+# Helper: create a fresh in-memory DuckDB connection and (re)create parquet-backed views.
+def create_fresh_duckdb_with_views():
+    """
+    Returns a fresh in-memory DuckDB connection with httpfs enabled and the
+    all_downloads / one_year_rolling views created from the remote parquet URLs.
+    Caller must close the returned connection.
+    """
+    local_con = duckdb.connect(database=":memory:", read_only=False)
+    try:
+        # try to install/load httpfs if necessary; ignore errors if preinstalled
+        try:
+            local_con.execute("INSTALL httpfs;")
+            local_con.execute("LOAD httpfs;")
+        except Exception:
+            pass
+
+        # keep HF Spaces behavior consistent
+        try:
+            local_con.execute("SET enable_http_metadata_cache = false;")
+            local_con.execute("SET enable_object_cache = false;")
+        except Exception:
+            pass
+
+        # create views referencing remote parquet files
+        local_con.execute(f"""
+            CREATE OR REPLACE VIEW all_downloads AS
+            SELECT * FROM read_parquet('{hf_parquet_url_1}')
+        """)
+        local_con.execute(f"""
+            CREATE OR REPLACE VIEW one_year_rolling AS
+            SELECT * FROM read_parquet('{hf_parquet_url_2}')
+        """)
+    except Exception:
+        # If view creation fails, ensure connection is still returned for caller to handle/close
+        pass
+    return local_con
+
 # Query for most recent date in all_downloads
 def get_last_updated():
     try:
-        result = con.execute(
-            "SELECT MAX(time) as max_time FROM all_downloads"
-        ).fetchdf()
+        conn = create_fresh_duckdb_with_views()
+        try:
+            result = conn.execute("SELECT MAX(time) as max_time FROM all_downloads").fetchdf()
+        finally:
+            conn.close()
+
         max_time = result["max_time"].iloc[0]
         if pd.isnull(max_time):
             return "N/A"
@@ -26,30 +71,6 @@ def get_last_updated():
         return "N/A"
 
 
-def load_parquet_to_duckdb(con, parquet_url, view_name):
-    """
-    Loads a parquet file from a remote URL into DuckDB as a view.
-    Returns (start_dt, end_dt) for the 'time' column.
-    """
-    # Install and load httpfs extension for remote file access
-    con.execute("INSTALL httpfs;")
-    con.execute("LOAD httpfs;")
-
-    # Create a view that references the remote parquet file
-    con.execute(f"""
-        CREATE OR REPLACE VIEW {view_name} AS 
-        SELECT * FROM read_parquet('{parquet_url}')
-    """)
-
-    # Get time range for slider
-    time_range = con.execute(
-        f"SELECT MIN(time) as min_time, MAX(time) as max_time FROM {view_name}"
-    ).fetchdf()
-    start_dt = pd.to_datetime(time_range["min_time"].iloc[0])
-    end_dt = pd.to_datetime(time_range["max_time"].iloc[0])
-    return start_dt, end_dt
-
-
 # DuckDB connection (global)
 con = duckdb.connect(database=":memory:", read_only=False)
 
@@ -66,12 +87,14 @@ print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}"
 try:
     overall_start_time = time.time()
 
-    # Load both parquet files as views
-    start_dt, end_dt = load_parquet_to_duckdb(con, hf_parquet_url_1, "all_downloads")
-    # Example: load a second parquet file as another view
-    start_dt2, end_dt2 = load_parquet_to_duckdb(
-        con, hf_parquet_url_2, "one_year_rolling"
-    )
+    # Create fresh connection, views, and read start/end time
+    conn = create_fresh_duckdb_with_views()
+    try:
+        time_range = conn.execute("SELECT MIN(time) as min_time, MAX(time) as max_time FROM all_downloads").fetchdf()
+        start_dt = pd.to_datetime(time_range["min_time"].iloc[0])
+        end_dt = pd.to_datetime(time_range["max_time"].iloc[0])
+    finally:
+        conn.close()
 
     msg = f"Successfully connected to datasets in {time.time() - overall_start_time:.2f}s."
     print(msg)
@@ -753,151 +776,164 @@ def _get_filtered_top_n_from_duckdb(
       - percent_of_total (percent of total across all returned model deltas)
     """
 
-    # Compute date window (if slider_value provided, use it; otherwise cover full range)
-    if slider_value and len(slider_value) == 2:
-        start = pd.to_datetime(slider_value[0], unit="s")
-        end = pd.to_datetime(slider_value[1], unit="s")
-    else:
-        start = pd.to_datetime("1970-01-01")
-        end = end_dt  # defined near top of file when parquet was loaded
-
-    start_str = str(start)
-    end_str = str(end)
-
-    # If grouping by country, transform some country values
-    if group_col == "org_country_single":
-        group_expr = """CASE 
-            WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-            WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-            ELSE org_country_single
-        END"""
-    else:
-        group_expr = group_col
+    # Create a fresh connection and load parquet-backed views for each call
+    local_con = create_fresh_duckdb_with_views()
 
-    # Derived-author requires author->country lookup; build separate SQL for that case
-    if group_col == "derived_author":
-        query = f"""
-        WITH base_data AS (
-            SELECT
-                {group_expr} AS group_key,
-                CASE 
-                    WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                    WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                    ELSE org_country_single
-                END AS org_country_single,
-                author,
-                derived_author,
-                merged_country_groups_single,
-                merged_modality,
-                model,
-                time,
-                downloadsAllTime
-            FROM {view}
-        ),
+    try:
+        # Compute date window (if slider_value provided, use it; otherwise cover full range)
+        if slider_value and len(slider_value) == 2:
+            start = pd.to_datetime(slider_value[0], unit="s")
+            end = pd.to_datetime(slider_value[1], unit="s")
+        else:
+            start = pd.to_datetime("1970-01-01")
+            # keep previous behavior if end_dt exists
+            try:
+                end_local = end_dt  # may be defined from initial load
+            except NameError:
+                end_local = pd.Timestamp.now()
+            end = end_local
+
+        start_str = str(start)
+        end_str = str(end)
+
+        # If grouping by country, transform some country values
+        if group_col == "org_country_single":
+            group_expr = """CASE 
+                WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
+                WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
+                ELSE org_country_single
+            END"""
+        else:
+            group_expr = group_col
+
+        # Derived-author requires author->country lookup; build separate SQL for that case
+        if group_col == "derived_author":
+            query = f"""
+            WITH base_data AS (
+                SELECT
+                    {group_expr} AS group_key,
+                    CASE 
+                        WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
+                        WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
+                        ELSE org_country_single
+                    END AS org_country_single,
+                    author,
+                    derived_author,
+                    merged_country_groups_single,
+                    merged_modality,
+                    model,
+                    time,
+                    downloadsAllTime
+                FROM {view}
+            ),
 
-        author_country_lookup AS (
-            SELECT DISTINCT
-                author,
-                FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
-            FROM base_data
-            WHERE author IS NOT NULL
-        ),
+            author_country_lookup AS (
+                SELECT DISTINCT
+                    author,
+                    FIRST_VALUE(org_country_single) OVER (PARTITION BY author ORDER BY downloadsAllTime DESC) AS author_country
+                FROM base_data
+                WHERE author IS NOT NULL
+            ),
 
-        model_metrics AS (
-            SELECT
-                model,
-                group_key,
-                ANY_VALUE(org_country_single) AS org_country_single,
-                ANY_VALUE(author) AS author,
-                ANY_VALUE(derived_author) AS derived_author,
-                ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
-                ANY_VALUE(merged_modality) AS merged_modality,
-                COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
-                - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
-                AS total_downloads
-            FROM base_data
-            GROUP BY model, group_key
-        ),
+            model_metrics AS (
+                SELECT
+                    model,
+                    group_key,
+                    ANY_VALUE(org_country_single) AS org_country_single,
+                    ANY_VALUE(author) AS author,
+                    ANY_VALUE(derived_author) AS derived_author,
+                    ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
+                    ANY_VALUE(merged_modality) AS merged_modality,
+                    COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
+                    - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
+                    AS total_downloads
+                FROM base_data
+                GROUP BY model, group_key
+            ),
 
-        total_downloads_cte AS (
-            SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
-        )
+            total_downloads_cte AS (
+                SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
+            )
 
-        SELECT
-            mm.model,
-            mm.group_key,
-            COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
-            mm.author,
-            mm.derived_author,
-            mm.merged_country_groups_single,
-            mm.merged_modality,
-            mm.total_downloads,
-            CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
-        FROM model_metrics mm
-        LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
-        CROSS JOIN total_downloads_cte td
-        WHERE mm.total_downloads > 0
-        ORDER BY mm.total_downloads DESC
-        LIMIT {top_n * 10};
-        """
-    else:
-        query = f"""
-        WITH base_data AS (
             SELECT
-                {group_expr} AS group_key,
-                CASE 
-                    WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
-                    WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
-                    ELSE org_country_single
-                END AS org_country_single,
-                author,
-                derived_author,
-                merged_country_groups_single,
-                merged_modality,
-                model,
-                time,
-                downloadsAllTime
-            FROM {view}
-        ),
+                mm.model,
+                mm.group_key,
+                COALESCE(acl.author_country, mm.org_country_single) AS org_country_single,
+                mm.author,
+                mm.derived_author,
+                mm.merged_country_groups_single,
+                mm.merged_modality,
+                mm.total_downloads,
+                CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
+            FROM model_metrics mm
+            LEFT JOIN author_country_lookup acl ON mm.group_key = acl.author
+            CROSS JOIN total_downloads_cte td
+            WHERE mm.total_downloads > 0
+            ORDER BY mm.total_downloads DESC
+            LIMIT {top_n * 10};
+            """
+        else:
+            query = f"""
+            WITH base_data AS (
+                SELECT
+                    {group_expr} AS group_key,
+                    CASE 
+                        WHEN org_country_single IN ('HF', 'United States of America') THEN 'United States of America'
+                        WHEN org_country_single IN ('International', 'Online', 'Online?') THEN 'International/Online'
+                        ELSE org_country_single
+                    END AS org_country_single,
+                    author,
+                    derived_author,
+                    merged_country_groups_single,
+                    merged_modality,
+                    model,
+                    time,
+                    downloadsAllTime
+                FROM {view}
+            ),
 
-        model_metrics AS (
-            SELECT
-                model,
-                group_key,
-                ANY_VALUE(org_country_single) AS org_country_single,
-                ANY_VALUE(author) AS author,
-                ANY_VALUE(derived_author) AS derived_author,
-                ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
-                ANY_VALUE(merged_modality) AS merged_modality,
-                COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
-                - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
-                AS total_downloads
-            FROM base_data
-            GROUP BY model, group_key
-        ),
+            model_metrics AS (
+                SELECT
+                    model,
+                    group_key,
+                    ANY_VALUE(org_country_single) AS org_country_single,
+                    ANY_VALUE(author) AS author,
+                    ANY_VALUE(derived_author) AS derived_author,
+                    ANY_VALUE(merged_country_groups_single) AS merged_country_groups_single,
+                    ANY_VALUE(merged_modality) AS merged_modality,
+                    COALESCE(MAX(CASE WHEN time <= '{end_str}' THEN downloadsAllTime END), 0)
+                    - COALESCE(MAX(CASE WHEN time < '{start_str}' THEN downloadsAllTime END), 0)
+                    AS total_downloads
+                FROM base_data
+                GROUP BY model, group_key
+            ),
 
-        total_downloads_cte AS (
-            SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
-        )
+            total_downloads_cte AS (
+                SELECT SUM(total_downloads) AS total_downloads_all FROM model_metrics
+            )
 
-        SELECT
-            mm.model,
-            mm.group_key,
-            mm.org_country_single,
-            mm.author,
-            mm.derived_author,
-            mm.merged_country_groups_single,
-            mm.merged_modality,
-            mm.total_downloads,
-            CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
-        FROM model_metrics mm
-        CROSS JOIN total_downloads_cte td
-        WHERE mm.total_downloads > 0
-        ORDER BY mm.total_downloads DESC
-        LIMIT {top_n * 10};
-        """
-
-    return con.execute(query).fetchdf()
+            SELECT
+                mm.model,
+                mm.group_key,
+                mm.org_country_single,
+                mm.author,
+                mm.derived_author,
+                mm.merged_country_groups_single,
+                mm.merged_modality,
+                mm.total_downloads,
+                CASE WHEN td.total_downloads_all = 0 THEN 0 ELSE ROUND(mm.total_downloads * 100.0 / td.total_downloads_all, 2) END AS percent_of_total
+            FROM model_metrics mm
+            CROSS JOIN total_downloads_cte td
+            WHERE mm.total_downloads > 0
+            ORDER BY mm.total_downloads DESC
+            LIMIT {top_n * 10};
+            """
+
+        # execute using the fresh local connection
+        result_df = local_con.execute(query).fetchdf()
+        return result_df
+    finally:
+        local_con.close()
 
 
 def _leaderboard_callback_logic(
@@ -991,7 +1027,7 @@ def update_top_countries(
         default_label="▼ Show Top 50",
         chip_color="#F0F9FF",
         view=selected_view,
-        derived_author_toggle=(attribution_type == "uploader"),
+        derived_author_toggle=(attribution_type == "original_creator"),
     )
 
 
@@ -1007,8 +1043,8 @@ def update_top_countries(
 def update_top_developers(
     n_clicks, slider_value, selected_view, attribution_type, current_label
 ):
-    # Use derived_author if attribution_type == "uploader", else author
-    group_col = "derived_author" if attribution_type == "uploader" else "author"
+    # Use derived_author if attribution_type == "original_creator", else author
+    group_col = "derived_author" if attribution_type == "original_creator" else "author"
     return _leaderboard_callback_logic(
         n_clicks,
         slider_value,
@@ -1018,7 +1054,7 @@ def update_top_developers(
         default_label="▼ Show Top 50",
         chip_color="#F0F9FF",
         view=selected_view,
-        derived_author_toggle=(attribution_type == "uploader"),
+        derived_author_toggle=(attribution_type == "original_creator"),
     )
 
 
@@ -1043,7 +1079,7 @@ def update_top_models(
         default_label="▼ Show More",
         chip_color="#F0F9FF",
         view=selected_view,
-        derived_author_toggle=(attribution_type == "uploader"),
+        derived_author_toggle=(attribution_type == "original_creator"),
     )
 
 

graphs/leaderboard.py

@@ -4,6 +4,7 @@ from dash_iconify import DashIconify
 import dash_mantine_components as dmc
 import base64
 import countryflag
+import duckdb
 
 button_style = {
     "display": "inline-block",
@@ -458,12 +459,51 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10, derived_author_toggl
     return display_for_render, download_top
 
 
+# Add dataset URLs used to create views when running queries from this module
+hf_parquet_url_1 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/all_downloads_with_annotations.parquet"
+hf_parquet_url_2 = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/one_year_rolling.parquet"
+
+
+def create_fresh_duckdb_with_views():
+    """
+    Returns a fresh in-memory DuckDB connection with httpfs enabled and the
+    all_downloads / one_year_rolling views created from the remote parquet URLs.
+    Caller must close the returned connection.
+    """
+    local_con = duckdb.connect(database=":memory:", read_only=False)
+    try:
+        try:
+            local_con.execute("INSTALL httpfs;")
+            local_con.execute("LOAD httpfs;")
+        except Exception:
+            pass
+        try:
+            local_con.execute("SET enable_http_metadata_cache = false;")
+            local_con.execute("SET enable_object_cache = false;")
+        except Exception:
+            pass
+
+        local_con.execute(f"""
+            CREATE OR REPLACE VIEW all_downloads AS
+            SELECT * FROM read_parquet('{hf_parquet_url_1}')
+        """)
+        local_con.execute(f"""
+            CREATE OR REPLACE VIEW one_year_rolling AS
+            SELECT * FROM read_parquet('{hf_parquet_url_2}')
+        """)
+    except Exception:
+        pass
+    return local_con
+
+
 def get_top_n_from_duckdb(
     con, group_col, top_n=10, time_filter=None, view="all_downloads"
 ):
     """
     Query DuckDB directly to get model-level rows with per-model total_downloads (delta or full)
     Returns rows similar to _get_filtered_top_n_from_duckdb in app.py.
+    NOTE: This function now opens a fresh DuckDB connection internally and ignores
+    any external connection passed in. Keep signature for compatibility.
     """
     # Compute date window
     if time_filter and len(time_filter) == 2:
@@ -610,11 +650,15 @@ def get_top_n_from_duckdb(
         LIMIT {top_n * 10};
         """
 
+    # Open a fresh in-memory connection that creates the views, run the query, close.
+    conn_local = create_fresh_duckdb_with_views()
     try:
-        return con.execute(query).fetchdf()
+        return conn_local.execute(query).fetchdf()
     except Exception as e:
         print(f"Error querying DuckDB: {e}")
         return pd.DataFrame()
+    finally:
+        conn_local.close()
 
 
 def format_large_number(n):