File size: 779 Bytes
6eed659 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | import os, requests
from huggingface_hub import get_token
tok=get_token(); h={"Authorization":f"Bearer {tok}"}
os.makedirs("/scratch/data/urban_noise/large", exist_ok=True)
base="https://huggingface.co/datasets/Sunbird/urban-noise-uganda-61k/resolve/main/large/"
for i in range(6):
fn=f"train-{i:05d}-of-00006.parquet"
dst=f"/scratch/data/urban_noise/large/{fn}"
if os.path.exists(dst) and os.path.getsize(dst)>1e6:
print(f"{fn} deja la"); continue
r=requests.get(base+fn, headers=h, timeout=300, stream=True, allow_redirects=True)
if not r.ok: print(f"{fn} ECHEC {r.status_code}"); continue
n=0
with open(dst,"wb") as f:
for c in r.iter_content(1<<20): f.write(c); n+=len(c)
print(f"{fn}: {n/1e6:.0f} MB")
print("SUNBIRD_ALL_OK")
|