Update app.py

app.py

@@ -1,34 +1,31 @@
-# --- START OF FINAL, POLISHED FILE app.py ---
+# --- app.py (Dataverse Explorer) ---
 
 import gradio as gr
 import pandas as pd
 import plotly.express as px
 import time
 from datasets import load_dataset
-# Using the stable, community-built RangeSlider component
-from gradio_rangeslider import RangeSlider
 
 # --- Constants ---
-PARAM_CHOICES = ['< 1B', '1B', '5B', '12B', '32B', '64B', '128B', '256B', '> 500B']
-PARAM_CHOICES_DEFAULT_INDICES = (0, len(PARAM_CHOICES) - 1)
-
 TOP_K_CHOICES = list(range(5, 51, 5))
-HF_DATASET_ID = "evijit/modelverse_daily_data"
-TAG_FILTER_CHOICES = [ "Audio & Speech", "Time series", "Robotics", "Music", "Video", "Images", "Text", "Biomedical", "Sciences" ]
-PIPELINE_TAGS = [ 'text-generation', 'text-to-image', 'text-classification', 'text2text-generation', 'audio-to-audio', 'feature-extraction', 'image-classification', 'translation', 'reinforcement-learning', 'fill-mask', 'text-to-speech', 'automatic-speech-recognition', 'image-text-to-text', 'token-classification', 'sentence-similarity', 'question-answering', 'image-feature-extraction', 'summarization', 'zero-shot-image-classification', 'object-detection', 'image-segmentation', 'image-to-image', 'image-to-text', 'audio-classification', 'visual-question-answering', 'text-to-video', 'zero-shot-classification', 'depth-estimation', 'text-ranking', 'image-to-video', 'multiple-choice', 'unconditional-image-generation', 'video-classification', 'text-to-audio', 'time-series-forecasting', 'any-to-any', 'video-text-to-text', 'table-question-answering' ]
-
-
-def load_models_data():
-    overall_start_time = time.time()
+HF_DATASET_ID = "evijit/dataverse_daily_data" # <-- Changed to the new dataset repo
+TAG_FILTER_CHOICES = [ 
+    "None", "Audio & Speech", "Time series", "Robotics", "Music", 
+    "Video", "Images", "Text", "Biomedical", "Sciences" 
+]
+
+def load_datasets_data():
+    """Load the processed datasets data from the Hugging Face Hub."""
+    start_time = time.time()
     print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
     try:
+        # Load the dataset from the Hub
         dataset_dict = load_dataset(HF_DATASET_ID)
+        # Convert the first split (usually 'train') to a pandas DataFrame
         df = dataset_dict[list(dataset_dict.keys())[0]].to_pandas()
-        if 'params' in df.columns:
-            df['params'] = pd.to_numeric(df['params'], errors='coerce').fillna(0)
-        else:
-            df['params'] = 0
-        msg = f"Successfully loaded dataset in {time.time() - overall_start_time:.2f}s."
+        
+        # No parameter processing needed for datasets
+        msg = f"Successfully loaded dataset in {time.time() - start_time:.2f}s."
         print(msg)
         return df, True, msg
     except Exception as e:
@@ -36,138 +33,135 @@ def load_models_data():
         print(err_msg)
         return pd.DataFrame(), False, err_msg
 
-def get_param_range_values(param_range_labels):
-    min_label, max_label = param_range_labels
-    min_val = 0.0 if '<' in min_label else float(min_label.replace('B', ''))
-    max_val = float('inf') if '>' in max_label else float(max_label.replace('B', ''))
-    return min_val, max_val
-
-def make_treemap_data(df, count_by, top_k=25, tag_filter=None, pipeline_filter=None, param_range=None, skip_orgs=None):
-    if df is None or df.empty: return pd.DataFrame()
+def make_treemap_data(df, count_by, top_k=25, tag_filter=None, skip_orgs=None):
+    """Filter and prepare data for the treemap visualization."""
+    if df is None or df.empty:
+        return pd.DataFrame()
+        
     filtered_df = df.copy()
-    col_map = { "Audio & Speech": "is_audio_speech", "Music": "has_music", "Robotics": "has_robot", "Biomedical": "is_biomed", "Time series": "has_series", "Sciences": "has_science", "Video": "has_video", "Images": "has_image", "Text": "has_text" }
-    if tag_filter and tag_filter in col_map and col_map[tag_filter] in filtered_df.columns:
-        filtered_df = filtered_df[filtered_df[col_map[tag_filter]]]
-    if pipeline_filter and "pipeline_tag" in filtered_df.columns:
-        filtered_df = filtered_df[filtered_df["pipeline_tag"].astype(str) == pipeline_filter]
-    if param_range:
-        min_params, max_params = get_param_range_values(param_range)
-        is_default_range = (param_range[0] == PARAM_CHOICES[0] and param_range[1] == PARAM_CHOICES[-1])
-        if not is_default_range and 'params' in filtered_df.columns:
-            if min_params is not None: filtered_df = filtered_df[filtered_df['params'] >= min_params]
-            if max_params is not None and max_params != float('inf'): filtered_df = filtered_df[filtered_df['params'] < max_params]
+    
+    # Map UI-friendly tag names to the boolean columns in the dataframe
+    col_map = { 
+        "Audio & Speech": "is_audio_speech", "Music": "has_music", "Robotics": "has_robot", 
+        "Biomedical": "is_biomed", "Time series": "has_series", "Sciences": "has_science", 
+        "Video": "has_video", "Images": "has_image", "Text": "has_text" 
+    }
+    
+    # Apply tag filter if a valid one is selected
+    if tag_filter and tag_filter != "None" and tag_filter in col_map:
+        if col_map[tag_filter] in filtered_df.columns:
+            filtered_df = filtered_df[filtered_df[col_map[tag_filter]]]
+
+    # Skip specified organizations if any are provided
     if skip_orgs and len(skip_orgs) > 0 and "organization" in filtered_df.columns:
         filtered_df = filtered_df[~filtered_df["organization"].isin(skip_orgs)]
-    if filtered_df.empty: return pd.DataFrame()
-    if count_by not in filtered_df.columns: filtered_df[count_by] = 0.0
+        
+    if filtered_df.empty:
+        return pd.DataFrame()
+        
+    # Ensure the metric column is numeric
+    if count_by not in filtered_df.columns:
+        filtered_df[count_by] = 0.0
     filtered_df[count_by] = pd.to_numeric(filtered_df[count_by], errors='coerce').fillna(0.0)
+    
+    # Group by organization and find the top K based on the selected metric
     org_totals = filtered_df.groupby("organization")[count_by].sum().nlargest(top_k, keep='first')
     top_orgs_list = org_totals.index.tolist()
+    
+    # Prepare the final data structure for the treemap
     treemap_data = filtered_df[filtered_df["organization"].isin(top_orgs_list)][["id", "organization", count_by]].copy()
-    treemap_data["root"] = "models"
+    treemap_data["root"] = "datasets" # Set the root node for the treemap
     return treemap_data
 
 def create_treemap(treemap_data, count_by, title=None):
+    """Generate the Plotly treemap figure."""
     if treemap_data.empty:
+        # Create a placeholder figure if no data matches the filters
         fig = px.treemap(names=["No data matches filters"], parents=[""], values=[1])
         fig.update_layout(title="No data matches the selected filters", margin=dict(t=50, l=25, r=25, b=25))
         return fig
-    fig = px.treemap(treemap_data, path=["root", "organization", "id"], values=count_by, title=title, color_discrete_sequence=px.colors.qualitative.Plotly)
+        
+    # Create the main treemap
+    fig = px.treemap(treemap_data, path=["root", "organization", "id"], values=count_by, 
+                     title=title, color_discrete_sequence=px.colors.qualitative.Plotly)
     fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))
-    fig.update_traces(textinfo="label+value+percent root", hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>")
+    fig.update_traces(
+        textinfo="label+value+percent root", 
+        hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>"
+    )
     return fig
 
-# --- FINAL, CORRECTED CSS ---
-custom_css = """
-/* Hide the extra UI elements from the RangeSlider component */
-#param-slider-wrapper .head,
-#param-slider-wrapper div[data-testid="range-slider"] > span {
-    display: none !important;
-}
-
-/* 
-  THIS IS THE KEY FIX:
-  We target all the individual component containers (divs with class .block)
-  that are *direct children* of our custom-classed group.
-  
-  This removes the "box-in-a-box" effect by making the inner component
-  containers transparent. The parent gr.Group now acts as the single card,
-  which is exactly what we want.
-*/
-.model-parameters-group > .block {
-    background: none !important;
-    border: none !important;
-    box-shadow: none !important;
-}
-"""
-
-with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css) as demo:
-    models_data_state = gr.State(pd.DataFrame())
+# --- Gradio UI Blocks ---
+with gr.Blocks(title="🤗 Dataverse Explorer", fill_width=True) as demo:
+    datasets_data_state = gr.State(pd.DataFrame())
     loading_complete_state = gr.State(False)
     
     with gr.Row():
-        gr.Markdown("# 🤗 ModelVerse Explorer")
+        gr.Markdown("# 🤗 Dataverse Explorer")
 
     with gr.Row():
         with gr.Column(scale=1):
+            # --- Control Panel ---
+            count_by_dropdown = gr.Dropdown(
+                label="Metric", 
+                choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], 
+                value="downloads"
+            )
             
-            # This section remains un-grouped for a consistent flat look
-            count_by_dropdown = gr.Dropdown(label="Metric", choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], value="downloads")
-            filter_choice_radio = gr.Radio(label="Filter Type", choices=["None", "Tag Filter", "Pipeline Filter"], value="None")
+            tag_filter_dropdown = gr.Dropdown(
+                label="Filter by Tag", 
+                choices=TAG_FILTER_CHOICES, 
+                value="None"
+            )
             
-            tag_filter_dropdown = gr.Dropdown(label="Select Tag", choices=TAG_FILTER_CHOICES, value=None, visible=False)
-            pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
-            
-            # This group's styling will be modified by the custom CSS
-            with gr.Group(elem_classes="model-parameters-group"):
-                gr.Markdown("<div style='font-weight: 500;'>Model Parameters</div>")
-                param_range_slider = RangeSlider(
-                    minimum=0,
-                    maximum=len(PARAM_CHOICES) - 1,
-                    value=PARAM_CHOICES_DEFAULT_INDICES,
-                    step=1,
-                    label=None,
-                    show_label=False,
-                    elem_id="param-slider-wrapper"
-                )
-                param_range_display = gr.Markdown(f"Range: `{PARAM_CHOICES[0]}` to `{PARAM_CHOICES[-1]}`")
+            top_k_dropdown = gr.Dropdown(
+                label="Number of Top Organizations", 
+                choices=TOP_K_CHOICES, 
+                value=25
+            )
             
-            # This section remains un-grouped
-            top_k_dropdown = gr.Dropdown(label="Number of Top Organizations", choices=TOP_K_CHOICES, value=25)
-            skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski")
+            skip_orgs_textbox = gr.Textbox(
+                label="Organizations to Skip (comma-separated)", 
+                value="huggingface,google,facebook,microsoft,amazon"
+            )
             
-            generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False)
+            generate_plot_button = gr.Button(
+                value="Generate Plot", 
+                variant="primary", 
+                interactive=False
+            )
 
         with gr.Column(scale=3):
+            # --- Output Area ---
             plot_output = gr.Plot()
             status_message_md = gr.Markdown("Initializing...")
             data_info_md = gr.Markdown("")
     
-    def update_param_display(value: tuple):
-        min_idx, max_idx = int(value[0]), int(value[1])
-        return f"Range: `{PARAM_CHOICES[min_idx]}` to `{PARAM_CHOICES[max_idx]}`"
-    
-    param_range_slider.change(update_param_display, param_range_slider, param_range_display)
-
-    def _update_button_interactivity(is_loaded_flag): return gr.update(interactive=is_loaded_flag)
-    loading_complete_state.change(fn=_update_button_interactivity, inputs=loading_complete_state, outputs=generate_plot_button)
-
-    def _toggle_filters_visibility(choice): 
-        return gr.update(visible=choice == "Tag Filter"), gr.update(visible=choice == "Pipeline Filter")
-    filter_choice_radio.change(fn=_toggle_filters_visibility, inputs=filter_choice_radio, outputs=[tag_filter_dropdown, pipeline_filter_dropdown])
+    # --- Controller Functions ---
+    def _update_button_interactivity(is_loaded_flag):
+        """Enable the generate button once data is loaded."""
+        return gr.update(interactive=is_loaded_flag)
 
     def ui_load_data_controller(progress=gr.Progress()):
+        """Handles the initial data loading and updates the UI with status."""
         progress(0, desc=f"Loading dataset '{HF_DATASET_ID}'...")
         try:
-            current_df, load_success_flag, status_msg_from_load = load_models_data()
+            current_df, load_success_flag, status_msg_from_load = load_datasets_data()
             if load_success_flag:
                 progress(0.9, desc="Processing data...")
+                # Format the timestamp for display
                 date_display = "Pre-processed (date unavailable)"
                 if 'data_download_timestamp' in current_df.columns and pd.notna(current_df['data_download_timestamp'].iloc[0]):
                     ts = pd.to_datetime(current_df['data_download_timestamp'].iloc[0], utc=True)
                     date_display = ts.strftime('%B %d, %Y, %H:%M:%S %Z')
-                param_count = (current_df['params'] > 0).sum() if 'params' in current_df.columns else 0
-                data_info_text = f"### Data Information\n- Source: `{HF_DATASET_ID}`\n- Status: {status_msg_from_load}\n- Total models loaded: {len(current_df):,}\n- Models with parameter counts: {param_count:,}\n- Data as of: {date_display}\n"
+                
+                # Create the data information summary
+                data_info_text = (
+                    f"### Data Information\n- Source: `{HF_DATASET_ID}`\n"
+                    f"- Status: {status_msg_from_load}\n"
+                    f"- Total datasets loaded: {len(current_df):,}\n"
+                    f"- Data as of: {date_display}\n"
+                )
                 status_msg_ui = "Data loaded. Ready to generate plot."
             else:
                 data_info_text = f"### Data Load Failed\n- {status_msg_from_load}"
@@ -177,52 +171,61 @@ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css
             data_info_text = f"### Critical Error\n- {status_msg_ui}"
             load_success_flag = False
             print(f"Critical error in ui_load_data_controller: {e}")
+            
         return current_df, load_success_flag, data_info_text, status_msg_ui
 
-    def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice, 
-                                   param_range_indices, k_orgs, skip_orgs_input, df_current_models, progress=gr.Progress()):
-        if df_current_models is None or df_current_models.empty:
-            return create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded"), "Model data is not loaded."
+    def ui_generate_plot_controller(metric_choice, tag_choice, k_orgs, 
+                                   skip_orgs_input, df_current_datasets, progress=gr.Progress()):
+        """Handles the plot generation based on user inputs."""
+        if df_current_datasets is None or df_current_datasets.empty:
+            return create_treemap(pd.DataFrame(), metric_choice), "Dataset data is not loaded."
         
         progress(0.1, desc="Preparing data...")
-        tag_to_use = tag_choice if filter_type == "Tag Filter" else None
-        pipeline_to_use = pipeline_choice if filter_type == "Pipeline Filter" else None
         orgs_to_skip = [org.strip() for org in skip_orgs_input.split(',') if org.strip()]
         
-        min_label = PARAM_CHOICES[int(param_range_indices[0])]
-        max_label = PARAM_CHOICES[int(param_range_indices[1])]
-        param_labels_for_filtering = [min_label, max_label]
-        
-        treemap_df = make_treemap_data(df_current_models, metric_choice, k_orgs, tag_to_use, pipeline_to_use, param_labels_for_filtering, orgs_to_skip)
+        # Prepare data for the treemap
+        treemap_df = make_treemap_data(df_current_datasets, metric_choice, k_orgs, tag_choice, orgs_to_skip)
         
         progress(0.7, desc="Generating plot...")
+        # Create a user-friendly title for the chart
         title_labels = {"downloads": "Downloads (last 30 days)", "downloadsAllTime": "Downloads (All Time)", "likes": "Likes"}
-        chart_title = f"HuggingFace Models - {title_labels.get(metric_choice, metric_choice)} by Organization"
+        chart_title = f"HuggingFace Datasets - {title_labels.get(metric_choice, metric_choice)} by Organization"
         plotly_fig = create_treemap(treemap_df, metric_choice, chart_title)
         
+        # Generate summary statistics for the plot
         if treemap_df.empty:
             plot_stats_md = "No data matches the selected filters. Please try different options."
         else:
             total_items_in_plot = len(treemap_df['id'].unique())
             total_value_in_plot = treemap_df[metric_choice].sum()
-            plot_stats_md = f"## Plot Statistics\n- **Models shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}"
+            plot_stats_md = f"## Plot Statistics\n- **Datasets shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}"
+            
         return plotly_fig, plot_stats_md
 
+    # --- Event Wiring ---
+    
+    # When the app loads, trigger the data fetching process
     demo.load(
         fn=ui_load_data_controller, 
         inputs=[], 
-        outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md]
+        outputs=[datasets_data_state, loading_complete_state, data_info_md, status_message_md]
     )
 
+    # When the loading is complete, enable the "Generate Plot" button
+    loading_complete_state.change(
+        fn=_update_button_interactivity, 
+        inputs=loading_complete_state, 
+        outputs=generate_plot_button
+    )
+
+    # When the "Generate Plot" button is clicked, trigger the plot generation
     generate_plot_button.click(
         fn=ui_generate_plot_controller,
-        inputs=[count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
-                param_range_slider, top_k_dropdown, skip_orgs_textbox, models_data_state],
+        inputs=[count_by_dropdown, tag_filter_dropdown, top_k_dropdown, 
+                skip_orgs_textbox, datasets_data_state],
         outputs=[plot_output, status_message_md]
     )
 
 if __name__ == "__main__":
-    print(f"Application starting...")
-    demo.queue().launch()
-
-# --- END OF FINAL, POLISHED FILE app.py ---
+    print("Application starting...")
+    demo.queue().launch()