fosters commited on
Commit
a4bc270
·
verified ·
1 Parent(s): 5421075

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +41 -32
app.py CHANGED
@@ -19,8 +19,7 @@ os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")
19
 
20
  N_CPUS = os.cpu_count() or 2
21
  BATCH_SIZE = 64 # max clips per ONNX forward pass
22
- API_TIMEOUT = 15 # seconds per datasets-server attempt
23
- API_RETRIES = 2
24
  STREAMING_TIMEOUT = 45 # seconds before giving up on streaming fallback
25
 
26
 
@@ -84,41 +83,51 @@ def _parse_audio_urls(rows: list) -> list[str]:
84
  return urls
85
 
86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str], str]:
88
- """Try /rows then /first-rows endpoints with retry. Returns (urls, status)."""
 
89
  headers = {"Authorization": f"Bearer {token}"} if token else {}
90
- endpoints = [
91
  ("/rows", {"dataset": repo_id, "config": "default", "split": "train", "offset": 0, "length": n}),
92
  ("/first-rows", {"dataset": repo_id, "config": "default", "split": "train"}),
93
  ]
94
- last_err = ""
95
- for endpoint, params in endpoints:
96
- for attempt in range(1, API_RETRIES + 1):
97
- try:
98
- t0 = time.time()
99
- resp = requests.get(
100
- f"{DATASETS_SERVER}{endpoint}", params=params,
101
- headers=headers, timeout=API_TIMEOUT,
102
- )
103
- elapsed = int((time.time() - t0) * 1000)
104
- if not resp.ok:
105
- last_err = f"{endpoint} HTTP {resp.status_code} (attempt {attempt}, {elapsed}ms)"
106
- break # 4xx/5xx — no point retrying same endpoint
107
- key = "rows" if endpoint == "/rows" else "rows"
108
- rows = resp.json().get(key, [])
109
- if not rows:
110
- last_err = f"{endpoint} empty ({elapsed}ms)"
111
- break
112
- urls = _parse_audio_urls(rows[:n])
113
- if urls:
114
- return urls, f"{endpoint} {elapsed}ms"
115
- last_err = f"{endpoint} no src ({elapsed}ms)"
116
- break
117
- except requests.Timeout:
118
- last_err = f"{endpoint} timeout>{API_TIMEOUT}s (attempt {attempt})"
119
- except Exception as exc:
120
- last_err = f"{endpoint} {exc} (attempt {attempt})"
121
- return [], last_err
122
 
123
 
124
  def _download_audio(url: str, token: str | None, max_sec: int) -> tuple[np.ndarray, int, int]:
 
19
 
20
  N_CPUS = os.cpu_count() or 2
21
  BATCH_SIZE = 64 # max clips per ONNX forward pass
22
+ API_TIMEOUT = 12 # seconds per datasets-server attempt (one try per endpoint)
 
23
  STREAMING_TIMEOUT = 45 # seconds before giving up on streaming fallback
24
 
25
 
 
83
  return urls
84
 
85
 
86
+ def _try_endpoint(endpoint: str, params: dict, headers: dict) -> tuple[list[str], str]:
87
+ """Single request attempt. Returns (urls, status_str)."""
88
+ try:
89
+ t0 = time.time()
90
+ resp = requests.get(f"{DATASETS_SERVER}{endpoint}", params=params,
91
+ headers=headers, timeout=API_TIMEOUT)
92
+ elapsed = int((time.time() - t0) * 1000)
93
+ if not resp.ok:
94
+ return [], f"{endpoint} HTTP {resp.status_code} ({elapsed}ms)"
95
+ rows = resp.json().get("rows", [])
96
+ if not rows:
97
+ return [], f"{endpoint} empty ({elapsed}ms)"
98
+ urls = _parse_audio_urls(rows)
99
+ if urls:
100
+ return urls, f"{endpoint} {elapsed}ms"
101
+ return [], f"{endpoint} no src ({elapsed}ms)"
102
+ except requests.Timeout:
103
+ return [], f"{endpoint} timeout>{API_TIMEOUT}s"
104
+ except Exception as exc:
105
+ return [], f"{endpoint} {exc}"
106
+
107
+
108
  def _fetch_audio_urls(repo_id: str, n: int, token: str | None) -> tuple[list[str], str]:
109
+ """Fire /rows and /first-rows in parallel, return first successful result.
110
+ Uses shutdown(wait=False) so the losing request doesn't block the caller."""
111
  headers = {"Authorization": f"Bearer {token}"} if token else {}
112
+ calls = [
113
  ("/rows", {"dataset": repo_id, "config": "default", "split": "train", "offset": 0, "length": n}),
114
  ("/first-rows", {"dataset": repo_id, "config": "default", "split": "train"}),
115
  ]
116
+ ex = concurrent.futures.ThreadPoolExecutor(max_workers=2)
117
+ futs = {ex.submit(_try_endpoint, ep, params, headers): ep for ep, params in calls}
118
+ errs = []
119
+ try:
120
+ for fut in concurrent.futures.as_completed(futs, timeout=API_TIMEOUT + 2):
121
+ urls, status = fut.result()
122
+ if urls:
123
+ ex.shutdown(wait=False, cancel_futures=True)
124
+ return urls[:n], status
125
+ errs.append(status)
126
+ except concurrent.futures.TimeoutError:
127
+ errs.append("both endpoints timed out")
128
+ finally:
129
+ ex.shutdown(wait=False, cancel_futures=True)
130
+ return [], " | ".join(errs)
 
 
 
 
 
 
 
 
 
 
 
 
 
131
 
132
 
133
  def _download_audio(url: str, token: str | None, max_sec: int) -> tuple[np.ndarray, int, int]: