Upload app.py with huggingface_hub

app.py

@@ -49,8 +49,8 @@ def _embed(audio_array: np.ndarray, sr: int, max_sec: int) -> np.ndarray:
     return out.embeddings.squeeze().numpy()
 
 
-def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> list[str]:
-    """Get direct audio URLs via datasets-server — no parquet download needed."""
+def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str], str]:
+    """Get direct audio URLs via datasets-server. Returns (urls, debug_info)."""
     headers = {"Authorization": f"Bearer {token}"} if token else {}
     resp = requests.get(
         f"{DATASETS_SERVER}/rows",
@@ -58,13 +58,26 @@ def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> list[str]:
         headers=headers,
         timeout=30,
     )
-    resp.raise_for_status()
+    if not resp.ok:
+        return [], f"datasets-server {resp.status_code}"
+
+    rows = resp.json().get("rows", [])
+    if not rows:
+        return [], "datasets-server: empty rows"
+
+    # Inspect actual audio field structure for debugging
+    sample_audio = rows[0]["row"].get("audio", {})
+    audio_keys = list(sample_audio.keys()) if isinstance(sample_audio, dict) else type(sample_audio).__name__
+
     urls = []
-    for row in resp.json().get("rows", []):
+    for row in rows:
         audio = row["row"].get("audio", {})
         if isinstance(audio, dict) and "src" in audio:
             urls.append(audio["src"])
-    return urls
+
+    if not urls:
+        return [], f"datasets-server: no src in audio field (keys={audio_keys})"
+    return urls, f"datasets-server ok (audio keys={audio_keys})"
 
 
 def _download_one(url: str, token: str | None, max_sec: int) -> tuple[np.ndarray, dict]:
@@ -90,31 +103,57 @@ def _download_one(url: str, token: str | None, max_sec: int) -> tuple[np.ndarray
     }
 
 
+def _streaming_fallback(repo: str, n_samples: int, audio_sec: int, token: str | None) -> list[np.ndarray]:
+    """Fallback: use datasets streaming when datasets-server is unavailable."""
+    from datasets import load_dataset, Audio as HFAudio
+    ds = load_dataset(repo, split="train", streaming=True, token=token)
+    ds = ds.cast_column("audio", HFAudio(decode=False))
+    embs = []
+    for j, row in enumerate(ds):
+        if j >= n_samples:
+            break
+        raw = row["audio"]
+        audio_bytes = raw.get("bytes") or open(raw["path"], "rb").read()
+        audio_array, sr = sf.read(io.BytesIO(audio_bytes))
+        embs.append(_embed(audio_array, sr, audio_sec))
+    return embs
+
+
 def _process_repo(
     repo: str, n_samples: int, audio_sec: int, token: str | None
 ) -> tuple[str, np.ndarray | None, str]:
     t0 = time.time()
     try:
-        urls = _fetch_audio_urls(repo, n_samples, token)
-        if not urls:
-            return repo, None, f"{repo}: no audio URLs from datasets-server"
+        urls, api_debug = _fetch_audio_urls(repo, n_samples, token)
         fetch_ms = int((time.time() - t0) * 1000)
 
-        # Download audio files in parallel (I/O bound)
-        with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as ex:
-            futures = [ex.submit(_download_one, u, token, audio_sec) for u in urls]
-            results = [f.result() for f in futures]
-
-        embs = [r[0] for r in results]
-        # Log stats from first sample as representative
-        s = results[0][1]
-        total_ms = int((time.time() - t0) * 1000)
-        log = (
-            f"{repo.split('/')[-1]}: "
-            f"api={fetch_ms}ms dl={s['dl_ms']}ms "
-            f"decode={s['decode_ms']}ms embed={s['embed_ms']}ms "
-            f"total={total_ms}ms ({s['size_kb']}KB/file)"
-        )
+        if urls:
+            # Fast path: parallel downloads via datasets-server URLs
+            with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as ex:
+                futures = [ex.submit(_download_one, u, token, audio_sec) for u in urls]
+                results = [f.result() for f in futures]
+            embs = [r[0] for r in results]
+            s = results[0][1]
+            total_ms = int((time.time() - t0) * 1000)
+            log = (
+                f"{repo.split('/')[-1]}: "
+                f"api={fetch_ms}ms dl={s['dl_ms']}ms "
+                f"decode={s['decode_ms']}ms embed={s['embed_ms']}ms "
+                f"total={total_ms}ms ({s['size_kb']}KB/file) [{api_debug}]"
+            )
+        else:
+            # Slow fallback: streaming (downloads parquet shard)
+            t1 = time.time()
+            embs = _streaming_fallback(repo, n_samples, audio_sec, token)
+            total_ms = int((time.time() - t0) * 1000)
+            log = (
+                f"{repo.split('/')[-1]}: streaming fallback "
+                f"total={total_ms}ms [reason: {api_debug}]"
+            )
+
+        if not embs:
+            return repo, None, f"{repo}: no audio samples extracted"
+
         return repo, np.mean(embs, axis=0), log
 
     except Exception as exc: