Upload folder using huggingface_hub

app.py

@@ -91,19 +91,32 @@ TABLE_FIELDS      = EXTRACTION_FIELDS + PAPER_FIELDS + UNIPROT_FIELDS
 # ---------------------------------------------------------------------------
 def scan_grid():
     models, mins, thresholds, fields = set(), set(), set(), set()
-    pattern = os.path.join(GRID_BASE, "model=*", "min=*", "t=*_cluster.html")
-    for fpath in glob.glob(pattern):
-        parts = fpath.split(os.sep)
-        for p in parts:
-            if p.startswith("model="):
-                models.add(p.replace("model=", ""))
-            elif p.startswith("min="):
-                mins.add(p.replace("min=", ""))
-        fname = os.path.basename(fpath)
-        m = re.match(r"t=([\d.]+)_(.+)_cluster\.html$", fname)
-        if m:
-            thresholds.add(m.group(1))
-            fields.add(m.group(2))
+    for pattern in (
+        os.path.join(GRID_BASE, "model=*", "min=*", "t=*_ALL_FIELDS.csv.gz"),
+        os.path.join(GRID_BASE, "model=*", "min=*", "t=*_ALL_FIELDS.csv"),
+    ):
+        for fpath in glob.glob(pattern):
+            parts = fpath.split(os.sep)
+            for p in parts:
+                if p.startswith("model="):
+                    models.add(p.replace("model=", ""))
+                elif p.startswith("min="):
+                    mins.add(p.replace("min=", ""))
+            fname = os.path.basename(fpath)
+            m = re.match(r"t=([\d.]+)_ALL_FIELDS\.csv", fname)
+            if m:
+                thresholds.add(m.group(1))
+    # Discover fields from a sample ALL_FIELDS file
+    sample = next(iter(
+        glob.glob(os.path.join(GRID_BASE, "model=*", "min=*", "t=*_ALL_FIELDS.csv.gz")) +
+        glob.glob(os.path.join(GRID_BASE, "model=*", "min=*", "t=*_ALL_FIELDS.csv"))
+    ), None)
+    if sample:
+        try:
+            ext = ".gz" if sample.endswith(".gz") else None
+            fields = set(pd.read_csv(sample, compression="gzip" if ext else None, usecols=["field"])["field"].unique())
+        except Exception:
+            pass
     return (
         sorted(models),
         sorted(mins, key=float),
@@ -556,6 +569,7 @@ def proteins_tab():
             page_size=25,
             page_action="native",
             sort_action="native",
+            sort_by=[{"column_id": "pmid", "direction": "asc"}],
             filter_action="none",
             row_selectable="single",
             selected_rows=[],
@@ -593,10 +607,10 @@ def proteins_tab():
 def clustering_tab():
     return dbc.Container([
         dbc.Row([
-            make_dropdown("Model", "dd-model", MODELS, value=MODELS[0] if MODELS else None),
-            make_dropdown("Min community size", "dd-min", MINS, value=MINS[0] if MINS else None),
-            make_dropdown("Threshold", "dd-threshold", THRESHOLDS, value=THRESHOLDS[0] if THRESHOLDS else None),
-            make_dropdown("Field", "dd-field", CLUSTER_FIELDS, value=CLUSTER_FIELDS[0] if CLUSTER_FIELDS else None),
+            make_dropdown("Model", "dd-model", MODELS, value="cambridgeltl__SapBERT-from-PubMedBERT-fulltext" if "cambridgeltl__SapBERT-from-PubMedBERT-fulltext" in MODELS else (MODELS[0] if MODELS else None)),
+            make_dropdown("Min community size", "dd-min", MINS, value="2" if "2" in MINS else (MINS[0] if MINS else None)),
+            make_dropdown("Threshold", "dd-threshold", THRESHOLDS, value="0.8" if "0.8" in THRESHOLDS else (THRESHOLDS[0] if THRESHOLDS else None)),
+            make_dropdown("Field", "dd-field", CLUSTER_FIELDS, value="organism_source" if "organism_source" in CLUSTER_FIELDS else (CLUSTER_FIELDS[0] if CLUSTER_FIELDS else None)),
         ], className="mb-3 g-3"),
         dbc.Row([
             dbc.Col([
@@ -1066,7 +1080,7 @@ def evaluation_tab():
                             {"label": " Azoreductases", "value": "azoreductases"},
                             {"label": " SAMs",          "value": "sams"},
                         ],
-                        value="sams",
+                        value="azoreductases",
                         inline=True,
                         inputStyle={"marginRight": "4px"},
                         labelStyle={"marginRight": "16px"},
@@ -1976,7 +1990,7 @@ _CONDITION_COLS = [
 )
 def show_cluster_point_detail(click_data, field, model, min_val, threshold, plot_type):
     if not click_data:
-        return html.Div(), None
+        return dash.no_update, dash.no_update
 
     point = click_data["points"][0]
 
@@ -2452,7 +2466,6 @@ def _eval_bar_figure(stats_df, y_col, err_col, y_label, models):
     Output("eval-llm-graph", "figure"),
     Input("eval-group", "value"),
     Input("eval-model-checklist", "value"),
-    prevent_initial_call=True,
 )
 def update_eval_llm(group, models):
     if EVAL_LLM_DF.empty or not group or not models:
@@ -2474,7 +2487,6 @@ def update_eval_llm(group, models):
     Input("eval-group", "value"),
     Input("eval-model-checklist", "value"),
     Input("eval-nlp-metric", "value"),
-    prevent_initial_call=True,
 )
 def update_eval_nlp(group, models, metric):
     if EVAL_NLP_DF.empty or not group or not models or not metric:

upload_to_hf.py

@@ -1,10 +1,33 @@
 from huggingface_hub import HfApi
 
 api = HfApi()
+REPO = "richiam/ProtoPure"
+BASE_IGNORE = ["*.pyc", "__pycache__", ".git", "*.html", "data/clustering/**/*.png"]
+
+# Upload everything except clustering data
 api.upload_folder(
     folder_path="/data/ralmadamonter/llm_dashboard",
-    repo_id="richiam/ProtoPure",
+    repo_id=REPO,
     repo_type="space",
-    ignore_patterns=["*.pyc", "__pycache__", ".git", "data/clustering/"],
+    ignore_patterns=BASE_IGNORE + ["data/clustering/"],
 )
-print("Done")
+print("Phase 1 done (app + non-clustering data)")
+
+# Upload clustering data in per-model batches
+import os
+CLUSTERING_DIR = "/data/ralmadamonter/llm_dashboard/data/clustering"
+for entry in sorted(os.listdir(CLUSTERING_DIR)):
+    entry_path = os.path.join(CLUSTERING_DIR, entry)
+    if not os.path.isdir(entry_path):
+        continue
+    print(f"Uploading {entry}...")
+    api.upload_folder(
+        folder_path=entry_path,
+        repo_id=REPO,
+        repo_type="space",
+        path_in_repo=f"data/clustering/{entry}",
+        ignore_patterns=["*.html", "*.png"],
+    )
+    print(f"  Done: {entry}")
+
+print("All done")