Fix demo data download auth + avoid synthetic shadowing

app.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import netrc
 from pathlib import Path
 from typing import Dict, List, Tuple, Optional
 
@@ -26,12 +27,25 @@ HUB_REPO_ID = "wi-lab/lwm-spectro"
 
 def _get_hf_token() -> str | None:
     # Spaces / HF Hub tooling uses a few common names.
-    return (
+    token = (
         os.getenv("HF_TOKEN")
         or os.getenv("HF_HUB_TOKEN")
         or os.getenv("HUGGINGFACEHUB_API_TOKEN")
         or os.getenv("HF_API_TOKEN")
+        or os.getenv("HUGGINGFACE_TOKEN")
+        or os.getenv("HUGGINGFACE_ACCESS_TOKEN")
     )
+    if token:
+        return token
+
+    # If a token exists in ~/.netrc (common in some environments), use it.
+    try:
+        auth = netrc.netrc().authenticators("huggingface.co")
+        if auth and auth[2]:
+            return auth[2]
+    except Exception:
+        return None
+    return None
 
 
 HF_TOKEN = _get_hf_token()
@@ -141,38 +155,8 @@ def _normalize_sample(sample: Dict[str, object]) -> Dict[str, object]:
 
 
 def _create_dummy_dataset(base_path: Path, moe_path: Path) -> None:
-    """Create a tiny synthetic dataset so the Space can start even if hub download fails."""
-    print(f"[WARN] Creating synthetic demo dataset at {base_path}")
-    rng = np.random.default_rng(42)
-    samples: List[Dict[str, object]] = []
-    techs = ["LTE", "WiFi", "5G"]
-    snrs = ["SNR0dB", "SNR10dB", "SNR20dB"]
-    mods = ["QPSK", "16QAM", "64QAM"]
-    mobs = ["pedestrian", "vehicular"]
-
-    for i in range(30):
-        tech = techs[i % len(techs)]
-        snr = snrs[i % len(snrs)]
-        mob = mobs[i % len(mobs)]
-        mod = mods[i % len(mods)]
-        spectrogram = rng.normal(size=(128, 128)).astype(np.float32)
-        embedding = rng.normal(size=(128,)).astype(np.float32)
-        moe_embedding = rng.normal(size=(128,)).astype(np.float32)
-        samples.append(
-            {
-                "tech": tech,
-                "snr": snr,
-                "mod": mod,
-                "mob": mob,
-                "data": spectrogram,
-                "embedding": embedding,
-                "moe_embedding": moe_embedding,
-            }
-        )
-
-    torch.save(samples, base_path)
-    torch.save(samples, moe_path)
-    print(f"[INFO] Synthetic dataset written to {base_path} and {moe_path}")
+    """Deprecated: kept for backward compatibility, but avoided in production."""
+    raise RuntimeError("Synthetic on-disk dataset generation disabled")
 
 
 def _create_dummy_samples() -> List[Dict[str, object]]:
@@ -210,20 +194,41 @@ def _ensure_local_file(local_path: Path, hub_filename: str) -> Optional[Path]:
     """Ensure a file exists locally; try Hub download if missing."""
     if local_path.exists() and not _is_git_lfs_pointer(local_path):
         return local_path
+
+    # Prefer a stored token if present (Spaces sometimes have credentials available
+    # even when HF_TOKEN env var is not explicitly set).
+    token = HF_TOKEN or True
+
+    # Try multiple repo types because the artifact may live under a model or dataset repo.
+    for repo_type in ("model", "dataset"):
+        try:
+            cached = hf_hub_download(
+                repo_id=HUB_REPO_ID,
+                filename=hub_filename,
+                token=token,
+                repo_type=repo_type,
+            )
+            cached_path = Path(cached)
+            print(f"[INFO] Using cached Hub file for {hub_filename}: {cached_path} (repo_type={repo_type})")
+            return cached_path
+        except Exception as exc:
+            last_exc = exc
+
+    # Final fallback: try downloading from the Space repo itself (useful when artifacts are stored in Space).
     try:
         cached = hf_hub_download(
-            repo_id=HUB_REPO_ID,
+            repo_id="wi-lab/LWM-Spectro",
             filename=hub_filename,
-            token=HF_TOKEN,
-            repo_type="model",
+            token=token,
+            repo_type="space",
         )
         cached_path = Path(cached)
-        # Prefer using the cached path directly to avoid large file copies or
-        # failures if the repo directory is not writable.
-        print(f"[INFO] Using cached Hub file for {hub_filename}: {cached_path}")
+        print(f"[INFO] Using cached Space file for {hub_filename}: {cached_path}")
         return cached_path
     except Exception as exc:
-        print(f"[WARN] Could not download {hub_filename} from Hub ({exc}); continuing without it.")
+        print(
+            f"[WARN] Could not download {hub_filename} from Hub ({last_exc}) or Space repo ({exc}); continuing without it."
+        )
         return None
 
 
@@ -239,14 +244,13 @@ def load_augmented_samples() -> Tuple[List[Dict[str, object]], bool]:
         print(f"[WARN] MoE data missing; falling back to base data: {base_path}")
         return _safe_load_tensor(base_path), False
 
-    # Last resort: attempt on-disk synthetic data, otherwise in-memory.
-    try:
-        _create_dummy_dataset(DEMO_DATA_PATH, MOE_DATA_PATH)
-        print(f"[WARN] Using synthetic on-disk dataset in {APP_DIR}")
-        return _safe_load_tensor(DEMO_DATA_PATH), False
-    except Exception as exc:
-        print(f"[WARN] Could not create synthetic files ({exc}); using in-memory synthetic dataset")
-        return _create_dummy_samples(), False
+    # Last resort: in-memory synthetic data (keeps app alive, but clearly not the full demo dataset).
+    print(
+        "[WARN] Falling back to a tiny synthetic dataset (30 samples). "
+        "This usually means the real demo_data*.pt could not be downloaded. "
+        "If the Hub repo is private, add a Space secret named HF_TOKEN with read access."
+    )
+    return _create_dummy_samples(), False
 
 
 def load_data(mapping: Dict[str, object]):
@@ -890,6 +894,8 @@ mapping_info = load_joint_mapping()
 df, has_moe_embeddings = load_data(mapping_info)
 CLASS_LABELS = mapping_info["label_names"]
 
+DATASET_STATUS = f"Dataset loaded: {len(df)} samples | MoE embeddings: {'yes' if has_moe_embeddings else 'no'}"
+
 has_moe_column = df["moe_embedding"].apply(lambda x: x is not None)
 joint_eval_df = df[has_moe_column & df["joint_label_id"].notna()]
 
@@ -922,6 +928,7 @@ def update_modulation_choices(selected_tech: Optional[str]):
 
 with gr.Blocks(title="LWM-Spectro Lab") as demo:
     gr.Markdown("# 🔬 LWM-Spectro Interactive Demo")
+    gr.Markdown(f"**{DATASET_STATUS}**")
     gr.Markdown(
         """
 **Having trouble seeing plots/images?**