| from huggingface_hub import snapshot_download | |
| snapshot_download( | |
| repo_id="google/WaxalNLP", repo_type="dataset", | |
| local_dir="/scratch/data/waxal", | |
| allow_patterns=[ | |
| "data/ASR/lin/lin-train-*.parquet","data/ASR/lin/lin-validation-*.parquet", | |
| "data/ASR/sna/sna-train-*.parquet","data/ASR/sna/sna-validation-*.parquet", | |
| ], | |
| max_workers=8, | |
| ) | |
| print("DL_DONE") | |