feat(loader): prefer BigBio script loader; fallback to parquet; clearer errors

app.py

@@ -57,38 +57,46 @@ def _train_ner_lora(
     try:
         # Medical aliases -> BigBio NER configs
         alias_map = {
-            # Use Parquet conversion branch via HTTPS URLs, discovered dynamically
+            # BigBio script-based configs (preferred with datasets<3.0)
             "bc5cdr": ("bigbio/bc5cdr", "bigbio_ner"),
             "ncbi_disease": ("bigbio/ncbi_disease", "bigbio_ner"),
         }
         lower_spec = ds_spec.lower()
         if lower_spec in alias_map:
             repo_id, subset = alias_map[lower_spec]
-            log(f"Using alias mapping (parquet HTTPS): {ds_spec} -> {repo_id} [{subset}]")
-            api = HfApi()
-            # List files at the parquet conversion revision
-            files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", revision="refs/convert/parquet")
-            # Build HTTPS URLs for each split
-            def split_files(split: str):
-                prefix = f"{subset}/{split}-"
-                return [
-                    hf_hub_url(repo_id=repo_id, filename=path, repo_type="dataset", revision="refs/convert/parquet")
-                    for path in files
-                    if path.startswith(prefix) and path.endswith(".parquet")
-                ]
-            train_files = split_files("train")
-            val_files = split_files("validation") or split_files("valid") or split_files("dev")
-            test_files = split_files("test")
-            if not train_files:
-                raise RuntimeError("No train parquet files found for BigBio subset")
-            data_files = {
-                "train": train_files,
-            }
-            if val_files:
-                data_files["validation"] = val_files
-            if test_files:
-                data_files["test"] = test_files
-            ds = load_dataset("parquet", data_files=data_files)
+            # 1) Try script loader first (requires datasets<3.0)
+            try:
+                log(f"Trying BigBio script loader: load_dataset('{repo_id}', '{subset}')")
+                ds = load_dataset(repo_id, subset, trust_remote_code=True)
+            except Exception as e_script:
+                log(f"Script loader failed: {e_script}")
+                # 2) Fallback to Parquet discovery via HTTPS
+                log("Falling back to Parquet discovery via refs/convert/parquet")
+                api = HfApi()
+                files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", revision="refs/convert/parquet")
+                def split_files(split: str):
+                    shard_prefix = f"{subset}/{split}-"
+                    dir_prefix = f"{subset}/{split}/"
+                    out = []
+                    for path in files:
+                        if not path.endswith(".parquet"):
+                            continue
+                        if path.startswith(shard_prefix) or path.startswith(dir_prefix):
+                            out.append(
+                                hf_hub_url(repo_id=repo_id, filename=path, repo_type="dataset", revision="refs/convert/parquet")
+                            )
+                    return sorted(out)
+                train_files = split_files("train")
+                val_files = split_files("validation") or split_files("valid") or split_files("dev")
+                test_files = split_files("test")
+                if not train_files:
+                    raise RuntimeError("No train parquet files found for BigBio subset; merge PR to pin datasets<3.0 or choose another dataset")
+                data_files = {"train": train_files}
+                if val_files:
+                    data_files["validation"] = val_files
+                if test_files:
+                    data_files["test"] = test_files
+                ds = load_dataset("parquet", data_files=data_files)
         elif ":" in ds_spec:
             ds_name, ds_config = [s.strip() for s in ds_spec.split(":", 1)]
             # If loading from community repo, allow remote code