fix(dataset): list parquet via HfApi and load via HTTPS data_files; add dataset dropdown

app.py

@@ -5,7 +5,7 @@ from typing import Optional, Dict, Any
 
 import gradio as gr
 
-from huggingface_hub import HfApi, create_repo
+from huggingface_hub import HfApi, create_repo, hf_hub_url
 
 
 DEFAULT_BASE_MODEL = "dmis-lab/biobert-base-cased-v1.2"
@@ -57,19 +57,37 @@ def _train_ner_lora(
     try:
         # Medical aliases -> BigBio NER configs
         alias_map = {
-            # Use Parquet conversion branch via hf:// scheme
-            "bc5cdr": "hf://datasets/bigbio/bc5cdr@refs/convert/parquet/bigbio_ner",
-            "ncbi_disease": "hf://datasets/bigbio/ncbi_disease@refs/convert/parquet/bigbio_ner",
+            # Use Parquet conversion branch via HTTPS URLs, discovered dynamically
+            "bc5cdr": ("bigbio/bc5cdr", "bigbio_ner"),
+            "ncbi_disease": ("bigbio/ncbi_disease", "bigbio_ner"),
         }
         lower_spec = ds_spec.lower()
         if lower_spec in alias_map:
-            base = alias_map[lower_spec]
-            log(f"Using alias mapping (parquet): {ds_spec} -> {base}")
+            repo_id, subset = alias_map[lower_spec]
+            log(f"Using alias mapping (parquet HTTPS): {ds_spec} -> {repo_id} [{subset}]")
+            api = HfApi()
+            # List files at the parquet conversion revision
+            files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", revision="refs/convert/parquet")
+            # Build HTTPS URLs for each split
+            def split_files(split: str):
+                prefix = f"{subset}/{split}-"
+                return [
+                    hf_hub_url(repo_id=repo_id, filename=path, repo_type="dataset", revision="refs/convert/parquet")
+                    for path in files
+                    if path.startswith(prefix) and path.endswith(".parquet")
+                ]
+            train_files = split_files("train")
+            val_files = split_files("validation") or split_files("valid") or split_files("dev")
+            test_files = split_files("test")
+            if not train_files:
+                raise RuntimeError("No train parquet files found for BigBio subset")
             data_files = {
-                "train": f"{base}/train-*.parquet",
-                "validation": f"{base}/validation-*.parquet",
-                "test": f"{base}/test-*.parquet",
+                "train": train_files,
             }
+            if val_files:
+                data_files["validation"] = val_files
+            if test_files:
+                data_files["test"] = test_files
             ds = load_dataset("parquet", data_files=data_files)
         elif ":" in ds_spec:
             ds_name, ds_config = [s.strip() for s in ds_spec.split(":", 1)]