Buckets:
| """Download source datasets for the rebuilt data pipeline. | |
| Usage: | |
| python data_new/download_datasets.py --all | |
| python data_new/download_datasets.py --only skatingverse mmfs | |
| python data_new/download_datasets.py --only fs_jump3d --skip-videos | |
| FineFS videos live on Baidu Pan and are throttled to a crawl for anonymous | |
| downloads, so pulling them automatically requires a logged-in Baidu session. | |
| Set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with BAIDU_STOKEN) to your | |
| pan.baidu.com session before running -- see download_finefs()'s docstring. | |
| Without one of those set, FineFS videos fall back to manual instructions. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import shutil | |
| import subprocess | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| PROJECT_ROOT = Path(__file__).resolve().parents[2] | |
| DATA_ROOT = PROJECT_ROOT / "data" | |
| DATASETS = { | |
| "skatingverse": { | |
| "description": "28,579 RGB clips, 28 action classes (6 jump types x 4 rotations + 4 spins + NONE)", | |
| "use": "primary training + evaluation", | |
| "size": "~15 GB", | |
| }, | |
| "mmfs": { | |
| "description": "11,671 clips, 256 categories, skeleton data pre-extracted", | |
| "note": "skeleton only; RGB requires email to liusl@dlut.edu.cn", | |
| "use": "supplementary skeleton data + quality scores", | |
| "size": "~2 GB skeleton", | |
| }, | |
| "finefs": { | |
| "description": "1,167 solo performances, 2-4 min each, temporal segmentation + scores", | |
| "note": ( | |
| "videos (~46 GB) on Baidu Pan only; auto-downloaded via BaiduPCS-Go if " | |
| "BAIDU_COOKIES/BAIDU_BDUSS is set, else falls back to manual instructions" | |
| ), | |
| "use": "temporal boundary annotations + quality assessment labels", | |
| "size": "~48 GB full / ~1 GB without videos", | |
| }, | |
| "fs_jump3d": { | |
| "description": "3D pose dataset for figure skating jumps, 12-view motion capture", | |
| "use": "3D pose estimation validation + ground truth", | |
| "size": "~9.6 GB full / ~808 MB without videos", | |
| }, | |
| "fsd10": { | |
| "description": ( | |
| "1,484 clips, 10 classes incl. a real named jump combo (Triple Lutz-Triple " | |
| "Toeloop); skeleton only, no RGB (2017-2018 championship footage)" | |
| ), | |
| "note": ( | |
| "Baidu Pan only, no smaller/alternate mirror found; auto-downloaded via " | |
| "BaiduPCS-Go if BAIDU_COOKIES/BAIDU_BDUSS is set, else falls back to manual " | |
| "instructions. Not to be confused with the different, unlabeled-by-name 30-class " | |
| "'FSD' skeleton set mirrored on videotag.bj.bcebos.com from an unrelated 2021 " | |
| "PaddlePaddle competition -- this is the original 10-class dataset with named " | |
| "classes from the FSD-10 paper (Liu et al., Neurocomputing 2020)." | |
| ), | |
| "use": "supplementary skeleton data with an explicit jump-combination class", | |
| "size": "unconfirmed (not documented publicly)", | |
| }, | |
| } | |
| def _copy_tree_contents(src: Path, dst: Path, max_workers: int = 8) -> None: | |
| """Copy src's contents into dst, copying individual files in parallel. | |
| Plain shutil.copytree copies one file at a time, which is dominated by | |
| per-file syscall overhead when there are tens of thousands of small | |
| clips. Copying is I/O-bound, so a thread pool lets multiple copies | |
| overlap instead of waiting on each file in turn. | |
| """ | |
| dst.mkdir(parents=True, exist_ok=True) | |
| for child in src.iterdir(): | |
| target = dst / child.name | |
| if child.is_dir() and target.exists(): | |
| shutil.rmtree(target) | |
| files = [(p, dst / p.relative_to(src)) for p in src.rglob("*") if p.is_file()] | |
| for _, target in files: | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| with ThreadPoolExecutor(max_workers=max_workers) as executor: | |
| list(executor.map(lambda pair: shutil.copy2(*pair), files)) | |
| def _require_gdown(): | |
| try: | |
| import gdown | |
| return gdown | |
| except ImportError as exc: | |
| raise RuntimeError("Install gdown first: pip install gdown") from exc | |
| def _download_gdrive_folder(folder_id: str, output_dir: Path) -> None: | |
| gdown = _require_gdown() | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| url = f"https://drive.google.com/drive/folders/{folder_id}" | |
| gdown.download_folder(url, output=str(output_dir), quiet=False, use_cookies=False) | |
| def _download_gdrive_file(file_id: str, output_path: Path) -> None: | |
| gdown = _require_gdown() | |
| output_path.parent.mkdir(parents=True, exist_ok=True) | |
| gdown.download(id=file_id, output=str(output_path), quiet=False) | |
| _SKATINGVERSE_MODELSCOPE_NAMESPACE = "awei2003" | |
| _SKATINGVERSE_MODELSCOPE_DATASET = "1st_SkatingVerse_Dataset" | |
| _SKATINGVERSE_LABEL_FILES = ("train.txt", "mapping.txt", "answer.txt") | |
| def _download_skatingverse_labels(output_dir: Path) -> None: | |
| """Fetch train.txt/mapping.txt/answer.txt from ModelScope's OSS-backed storage. | |
| The Kaggle mirror only ships train_videos/ and test_videos/ with no label | |
| file (parent-folder name is the only "label" and doesn't parse). The real | |
| labels live in the dataset's ModelScope repo, but as OSS objects rather than | |
| git-tracked files, so they're listed via the OSS tree API and fetched from | |
| the presigned URLs it returns instead of a git clone. | |
| """ | |
| import requests | |
| if all((output_dir / name).exists() for name in _SKATINGVERSE_LABEL_FILES): | |
| print(" label files already present, skipping ModelScope fetch") | |
| return | |
| tree_url = ( | |
| f"https://www.modelscope.cn/api/v1/datasets/" | |
| f"{_SKATINGVERSE_MODELSCOPE_NAMESPACE}/{_SKATINGVERSE_MODELSCOPE_DATASET}/oss/tree" | |
| ) | |
| print(" fetching label files (train.txt, mapping.txt, answer.txt) from ModelScope") | |
| resp = requests.get(tree_url, params={"MaxLimit": 200, "Revision": "master", "Recursive": "true"}) | |
| resp.raise_for_status() | |
| entries = resp.json()["Data"] | |
| by_name = {entry["Key"]: entry["Url"] for entry in entries} | |
| missing = [name for name in _SKATINGVERSE_LABEL_FILES if name not in by_name] | |
| if missing: | |
| raise RuntimeError(f"ModelScope OSS listing is missing expected label file(s): {missing}") | |
| for name in _SKATINGVERSE_LABEL_FILES: | |
| dest = output_dir / name | |
| if dest.exists(): | |
| continue | |
| with requests.get(by_name[name], stream=True) as file_resp: | |
| file_resp.raise_for_status() | |
| with open(dest, "wb") as f: | |
| for chunk in file_resp.iter_content(chunk_size=1 << 20): | |
| f.write(chunk) | |
| print(f" downloaded {name}") | |
| def download_skatingverse(output_dir: Path, skip_videos: bool = False) -> Path: | |
| """Download SkatingVerse from Kaggle — actual RGB video clips. | |
| Also fetches train.txt/mapping.txt/answer.txt from ModelScope, since the | |
| Kaggle mirror doesn't include them (see _download_skatingverse_labels). | |
| The ModelScope label fetch hits a different host and doesn't depend on | |
| the Kaggle download, so it runs concurrently instead of after it. | |
| """ | |
| try: | |
| import kagglehub | |
| except ImportError as exc: | |
| raise RuntimeError("Install kagglehub first: pip install kagglehub") from exc | |
| with ThreadPoolExecutor(max_workers=2) as executor: | |
| labels_future = executor.submit(_download_skatingverse_labels, output_dir) | |
| downloaded = Path(kagglehub.dataset_download("elephantfish/skatingverse-dataset")) | |
| _copy_tree_contents(downloaded, output_dir) | |
| labels_future.result() | |
| return output_dir | |
| def download_mmfs(output_dir: Path, skip_videos: bool = False) -> Path: | |
| """Download MMFS skeleton data from Google Drive.""" | |
| gdown = _require_gdown() | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| gdrive_id = "12lhALM9u17-_V0pH8Epa7LgPJY4x3sM4" | |
| url = f"https://drive.google.com/uc?id={gdrive_id}" | |
| try: | |
| gdown.download_folder(url, output=str(output_dir), quiet=False, use_cookies=False) | |
| except Exception: | |
| output_file = output_dir / f"{gdrive_id}.download" | |
| gdown.download(id=gdrive_id, output=str(output_file), quiet=False) | |
| return output_dir | |
| _FINEFS_BAIDU_URL = "https://pan.baidu.com/s/1ihV47FIgNhATm5g1XTcaNg" | |
| _FINEFS_BAIDU_CODE = "hri6" | |
| _FINEFS_BAIDU_REMOTE_DIR = "/finefs_dataset" | |
| _FINEFS_EXT_TO_SUBDIR = { | |
| ".mp4": "videos", | |
| ".npz": "skeletons", | |
| ".pkl": "features", | |
| } | |
| def _baidupcs_go_bin() -> str: | |
| return os.environ.get("BAIDUPCS_GO_BIN", "BaiduPCS-Go") | |
| def _run_baidupcs(*args: str) -> subprocess.CompletedProcess: | |
| bin_path = _baidupcs_go_bin() | |
| try: | |
| return subprocess.run([bin_path, *args], capture_output=True, text=True) | |
| except FileNotFoundError as exc: | |
| raise RuntimeError( | |
| "BaiduPCS-Go not found. Install a release binary from " | |
| "https://github.com/qjfoidnh/BaiduPCS-Go/releases (or `go install " | |
| "github.com/qjfoidnh/BaiduPCS-Go@latest`), or point BAIDUPCS_GO_BIN " | |
| "at the binary." | |
| ) from exc | |
| def _baidupcs_login() -> None: | |
| """Log BaiduPCS-Go into a real Baidu account so it can transfer + download at full speed. | |
| Anonymous Baidu Pan downloads of large shared files are throttled to a few | |
| KB/s, so this needs a real session: log into pan.baidu.com in a browser, | |
| open devtools' Network tab, and copy the Cookie header of any | |
| pan.baidu.com request into BAIDU_COOKIES (or extract BDUSS/STOKEN from it | |
| into BAIDU_BDUSS/BAIDU_STOKEN). STOKEN is required for the transfer step. | |
| """ | |
| cookies = os.environ.get("BAIDU_COOKIES") | |
| bduss = os.environ.get("BAIDU_BDUSS") | |
| if cookies: | |
| result = _run_baidupcs("login", f"-cookies={cookies}") | |
| elif bduss: | |
| args = ["login", f"-bduss={bduss}"] | |
| stoken = os.environ.get("BAIDU_STOKEN") | |
| if stoken: | |
| args.append(f"-stoken={stoken}") | |
| result = _run_baidupcs(*args) | |
| else: | |
| raise RuntimeError( | |
| "Set BAIDU_COOKIES (or BAIDU_BDUSS[+BAIDU_STOKEN]) to a logged-in " | |
| "pan.baidu.com session to download from Baidu Pan automatically." | |
| ) | |
| if result.returncode != 0: | |
| raise RuntimeError(f"BaiduPCS-Go login failed:\n{result.stdout}\n{result.stderr}") | |
| def _sort_finefs_baidu_download(raw_dir: Path, output_dir: Path) -> None: | |
| """Route files pulled from the Baidu share into videos/skeletons/features by extension.""" | |
| for src in raw_dir.rglob("*"): | |
| if not src.is_file(): | |
| continue | |
| subdir = _FINEFS_EXT_TO_SUBDIR.get(src.suffix.lower()) | |
| if subdir is None: | |
| continue | |
| dst = output_dir / subdir / src.name | |
| dst.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.move(str(src), str(dst)) | |
| shutil.rmtree(raw_dir, ignore_errors=True) | |
| def _download_finefs_videos_via_baidupcs(output_dir: Path) -> None: | |
| """Transfer the FineFS Baidu share into our own account, then download it. | |
| Baidu only lets shared files be downloaded at full speed after they're | |
| "transferred" (saved) into a logged-in account's own drive -- anonymous | |
| downloads of a multi-GB shared file are capped to a crawl. This requires | |
| _baidupcs_login() to have set up a real session first. | |
| """ | |
| _run_baidupcs("mkdir", _FINEFS_BAIDU_REMOTE_DIR) | |
| _run_baidupcs("cd", _FINEFS_BAIDU_REMOTE_DIR) | |
| transfer = _run_baidupcs("transfer", _FINEFS_BAIDU_URL, _FINEFS_BAIDU_CODE, "-collect") | |
| if transfer.returncode != 0: | |
| raise RuntimeError(f"BaiduPCS-Go transfer failed:\n{transfer.stdout}\n{transfer.stderr}") | |
| raw_dir = output_dir / "_baidu_download" | |
| raw_dir.mkdir(parents=True, exist_ok=True) | |
| download = _run_baidupcs("download", _FINEFS_BAIDU_REMOTE_DIR, f"--saveto={raw_dir}") | |
| if download.returncode != 0: | |
| raise RuntimeError(f"BaiduPCS-Go download failed:\n{download.stdout}\n{download.stderr}") | |
| _sort_finefs_baidu_download(raw_dir, output_dir) | |
| def download_finefs(output_dir: Path, skip_videos: bool = False) -> Path: | |
| """Download FineFS annotations + skeleton data, and attempt videos via Baidu Pan. | |
| The repo itself only has documentation. The actual data lives on Baidu Pan: | |
| https://pan.baidu.com/s/1ihV47FIgNhATm5g1XTcaNg (code: hri6) | |
| The Google Drive link in the upstream README is broken (empty URL), and per | |
| the maintainer (github.com/yanliji/FineFS-dataset issue #4), there is no | |
| other mirror. Videos (~46 GB) are only available via Baidu Pan, whose | |
| anonymous downloads are throttled hard, so this only succeeds with | |
| BAIDU_COOKIES or BAIDU_BDUSS set (see _baidupcs_login docstring). If that | |
| isn't set, or BaiduPCS-Go isn't installed, or the transfer/download itself | |
| fails, this falls back to printing manual instructions instead of raising. | |
| """ | |
| repo_url = "https://github.com/yanliji/FineFS-dataset.git" | |
| repo_dir = output_dir / "_repo" | |
| if repo_dir.exists() and any(repo_dir.iterdir()): | |
| subprocess.run(["git", "-C", str(repo_dir), "pull", "--ff-only"], check=True) | |
| else: | |
| repo_dir.parent.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(["git", "clone", repo_url, str(repo_dir)], check=True) | |
| annotations_dir = output_dir / "annotations" | |
| for src in repo_dir.rglob("*.json"): | |
| dst = annotations_dir / src.relative_to(repo_dir) | |
| dst.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copy2(src, dst) | |
| for src in repo_dir.rglob("*.csv"): | |
| dst = annotations_dir / src.relative_to(repo_dir) | |
| dst.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copy2(src, dst) | |
| baidu_url = _FINEFS_BAIDU_URL | |
| baidu_code = _FINEFS_BAIDU_CODE | |
| if skip_videos: | |
| print(" skipping videos (--skip-videos)") | |
| else: | |
| videos_dir = output_dir / "videos" | |
| skeletons_dir = output_dir / "skeletons" | |
| if videos_dir.exists() and any(videos_dir.iterdir()): | |
| print(" videos directory already populated, skipping Baidu download") | |
| else: | |
| try: | |
| print(" attempting automated Baidu Pan download via BaiduPCS-Go...") | |
| _baidupcs_login() | |
| _download_finefs_videos_via_baidupcs(output_dir) | |
| print(f" downloaded videos/skeletons/features via BaiduPCS-Go -> {output_dir.resolve()}") | |
| except Exception as exc: | |
| print(f" automated download failed: {exc}") | |
| print(f" WARNING: FineFS videos (~46 GB) must be downloaded manually from Baidu Pan.") | |
| print(f" URL: {baidu_url}") | |
| print(f" Code: {baidu_code}") | |
| print(f" Place videos in: {videos_dir.resolve()}") | |
| print(f" Place skeletons in: {skeletons_dir.resolve()}") | |
| if not (output_dir / "videos").exists() or not any((output_dir / "videos").iterdir()): | |
| readme = output_dir / "DOWNLOAD_INSTRUCTIONS.txt" | |
| readme.write_text( | |
| f"FineFS Dataset\n" | |
| f"==============\n\n" | |
| f"Annotations were extracted from the GitHub repo.\n\n" | |
| f"Videos (~46 GB), skeletons (~828 MB), and features (~955 MB)\n" | |
| f"live on Baidu Pan:\n\n" | |
| f" URL: {baidu_url}\n" | |
| f" Extraction code: {baidu_code}\n\n" | |
| f"Option 1 (automated): set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with\n" | |
| f"BAIDU_STOKEN) to a logged-in pan.baidu.com session and re-run this download --\n" | |
| f"it will transfer + fetch the share via BaiduPCS-Go automatically. STOKEN is\n" | |
| f"required for the transfer step; without it, the login succeeds but transfer\n" | |
| f"will fail. See _baidupcs_login() in download.py for how to get these values.\n" | |
| f"Requires BaiduPCS-Go: https://github.com/qjfoidnh/BaiduPCS-Go\n\n" | |
| f"Option 2 (manual): download the share above by hand and place contents into\n" | |
| f"this directory:\n" | |
| f" videos/ -> .mp4 files\n" | |
| f" skeletons/ -> .npz files\n" | |
| f" features/ -> .pkl files\n\n" | |
| f"The Google Drive link in the upstream repo README is broken, and per the\n" | |
| f"maintainer (github.com/yanliji/FineFS-dataset issue #4) there is no other\n" | |
| f"mirror as of 2026.\n", | |
| encoding="utf-8", | |
| ) | |
| return output_dir | |
| _FS_JUMP3D_GDRIVE = { | |
| "c3d": { | |
| "folder_id": "1Ki9dxLuo78XFnCun9LGwWFlzO-A0FxJT", | |
| "description": "C3D motion capture files (302.6 MB)", | |
| }, | |
| "json": { | |
| "folder_id": "17gQJR-qzF_JTs8JZgwZc1wuRKvkwnwVj", | |
| "description": "JSON 3D pose files (505.2 MB)", | |
| }, | |
| "videos": { | |
| "folder_id": "1yvZMmK4hvrvK5ykqzkr1d-yVmImz-NNJ", | |
| "description": "12-viewpoint video recordings (8.84 GB)", | |
| }, | |
| } | |
| def download_fs_jump3d(output_dir: Path, skip_videos: bool = False) -> Path: | |
| """Download FS-Jump3D 3D pose data from Google Drive. | |
| Clones the repo for utilities (format.py, visualization.ipynb), then | |
| downloads the actual data (C3D, JSON, videos) from Google Drive. | |
| """ | |
| repo_url = "https://github.com/ryota-skating/FS-Jump3D.git" | |
| repo_dir = output_dir / "_repo" | |
| if repo_dir.exists() and any(repo_dir.iterdir()): | |
| subprocess.run(["git", "-C", str(repo_dir), "pull", "--ff-only"], check=True) | |
| else: | |
| repo_dir.parent.mkdir(parents=True, exist_ok=True) | |
| subprocess.run(["git", "clone", repo_url, str(repo_dir)], check=True) | |
| utils_dst = output_dir / "utils" | |
| utils_src = repo_dir / "utils" | |
| if utils_src.exists(): | |
| _copy_tree_contents(utils_src, utils_dst) | |
| for key, info in _FS_JUMP3D_GDRIVE.items(): | |
| if key == "videos" and skip_videos: | |
| print(f" skipping {info['description']} (--skip-videos)") | |
| continue | |
| dest = output_dir / key | |
| if dest.exists() and any(dest.iterdir()): | |
| print(f" {key}/ already populated, skipping") | |
| continue | |
| print(f" downloading {info['description']}") | |
| _download_gdrive_folder(info["folder_id"], dest) | |
| return output_dir | |
| _FSD10_BAIDU_URL = "https://pan.baidu.com/s/1d8dIxesymX00n9xCIHnVNg" | |
| _FSD10_BAIDU_CODE = "c4oc" | |
| _FSD10_BAIDU_REMOTE_DIR = "/fsd10_dataset" | |
| def download_fsd10(output_dir: Path, skip_videos: bool = False) -> Path: | |
| """Download FSD-10 (skeleton-only, no RGB) from Baidu Pan. | |
| No GitHub repo or alternate mirror exists for this one (checked HuggingFace, ModelScope, | |
| OpenDataLab, Kaggle, Academic Torrents -- nothing). skip_videos is accepted for interface | |
| consistency with the other downloaders but has no effect: the dataset never had RGB video | |
| to begin with ("RGB datasets would not be provided for copyright reasons" per the | |
| maintainer's own PaddleVideo docs), so there's nothing to skip. | |
| Do not confuse this with the different, unrelated 30-class "FSD" skeleton set mirrored on | |
| videotag.bj.bcebos.com (from a 2021 PaddlePaddle/CCF competition) -- that one is a clean, | |
| ungated direct download, but its 30 labels have no publicly documented class names, which | |
| defeats the point of wanting FSD-10's real named classes (e.g. its combination-jump class). | |
| """ | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| try: | |
| print(" attempting automated Baidu Pan download via BaiduPCS-Go...") | |
| _baidupcs_login() | |
| _run_baidupcs("mkdir", _FSD10_BAIDU_REMOTE_DIR) | |
| _run_baidupcs("cd", _FSD10_BAIDU_REMOTE_DIR) | |
| transfer = _run_baidupcs("transfer", _FSD10_BAIDU_URL, _FSD10_BAIDU_CODE, "-collect") | |
| if transfer.returncode != 0: | |
| raise RuntimeError(f"BaiduPCS-Go transfer failed:\n{transfer.stdout}\n{transfer.stderr}") | |
| download = _run_baidupcs("download", _FSD10_BAIDU_REMOTE_DIR, f"--saveto={output_dir}") | |
| if download.returncode != 0: | |
| raise RuntimeError(f"BaiduPCS-Go download failed:\n{download.stdout}\n{download.stderr}") | |
| print(f" downloaded FSD-10 via BaiduPCS-Go -> {output_dir.resolve()}") | |
| except Exception as exc: | |
| print(f" automated download failed: {exc}") | |
| print(" WARNING: FSD-10 must be downloaded manually from Baidu Pan.") | |
| print(f" URL: {_FSD10_BAIDU_URL}") | |
| print(f" Code: {_FSD10_BAIDU_CODE}") | |
| print(f" Place downloaded contents into: {output_dir.resolve()}") | |
| readme = output_dir / "DOWNLOAD_INSTRUCTIONS.txt" | |
| readme.write_text( | |
| f"FSD-10 Dataset\n" | |
| f"==============\n\n" | |
| f"Skeleton-only (OpenPose-derived), 10 named classes incl. a real jump-combination\n" | |
| f"class (Triple Lutz-Triple Toeloop). No RGB video is provided by the maintainers.\n\n" | |
| f"Only available via Baidu Pan (no other mirror found):\n\n" | |
| f" URL: {_FSD10_BAIDU_URL}\n" | |
| f" Extraction code: {_FSD10_BAIDU_CODE}\n\n" | |
| f"Option 1 (automated): set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with\n" | |
| f"BAIDU_STOKEN) to a logged-in pan.baidu.com session and re-run this download --\n" | |
| f"it will transfer + fetch the share via BaiduPCS-Go automatically.\n" | |
| f"Requires BaiduPCS-Go: https://github.com/qjfoidnh/BaiduPCS-Go\n\n" | |
| f"Option 2 (manual): download the share above by hand into this directory.\n\n" | |
| f"Contact for questions: liusl@mail.dlut.edu.cn\n", | |
| encoding="utf-8", | |
| ) | |
| return output_dir | |
| DOWNLOADERS = { | |
| "skatingverse": download_skatingverse, | |
| "mmfs": download_mmfs, | |
| "finefs": download_finefs, | |
| "fsd10": download_fsd10, | |
| "fs_jump3d": download_fs_jump3d, | |
| } | |
| def download_dataset(name: str, data_root: Path = DATA_ROOT, skip_videos: bool = False) -> Path: | |
| """Download one configured dataset into data/{name}/.""" | |
| if name not in DATASETS: | |
| raise ValueError(f"Unknown dataset {name!r}. Choose from: {', '.join(DATASETS)}") | |
| output_dir = data_root / name | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| return DOWNLOADERS[name](output_dir, skip_videos=skip_videos) | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| group = parser.add_mutually_exclusive_group(required=True) | |
| group.add_argument("--all", action="store_true", help="Download all configured datasets") | |
| group.add_argument("--only", nargs="+", choices=sorted(DATASETS), help="Download selected datasets") | |
| parser.add_argument("--data-root", type=Path, default=DATA_ROOT, help="Output data root") | |
| parser.add_argument( | |
| "--skip-videos", action="store_true", | |
| help="Skip large video downloads (annotations and skeleton data only)", | |
| ) | |
| return parser.parse_args() | |
| def main() -> int: | |
| args = parse_args() | |
| selected = sorted(DATASETS) if args.all else args.only | |
| print(f"Downloading {len(selected)} dataset(s) into {args.data_root.resolve()}") | |
| if args.skip_videos: | |
| print("Video downloads will be skipped (--skip-videos)") | |
| successes = [] | |
| failures = [] | |
| for name in selected: | |
| dataset = DATASETS[name] | |
| print(f"\n{'=' * 60}") | |
| print(f"{name}: {dataset['description']}") | |
| print(f" use: {dataset['use']}") | |
| print(f" size: {dataset['size']}") | |
| if "note" in dataset: | |
| print(f" note: {dataset['note']}") | |
| try: | |
| output = download_dataset(name, args.data_root, skip_videos=args.skip_videos) | |
| successes.append((name, output)) | |
| print(f" -> {output.resolve()}") | |
| except Exception as exc: | |
| failures.append((name, exc)) | |
| print(f" FAILED: {exc}") | |
| print(f"\n{'=' * 60}") | |
| print("Summary") | |
| for name, output in successes: | |
| print(f" ok {name}: {output.resolve()}") | |
| for name, exc in failures: | |
| print(f" FAILED {name}: {exc}") | |
| return 1 if failures else 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 24 kB
- Xet hash:
- cff13dc799b4edd42c281fa625112cd421eee5a0073087ddab5733ec47feb381
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.