Buckets:

Rishik001's picture
download
raw
24 kB
"""Download source datasets for the rebuilt data pipeline.
Usage:
python data_new/download_datasets.py --all
python data_new/download_datasets.py --only skatingverse mmfs
python data_new/download_datasets.py --only fs_jump3d --skip-videos
FineFS videos live on Baidu Pan and are throttled to a crawl for anonymous
downloads, so pulling them automatically requires a logged-in Baidu session.
Set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with BAIDU_STOKEN) to your
pan.baidu.com session before running -- see download_finefs()'s docstring.
Without one of those set, FineFS videos fall back to manual instructions.
"""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_ROOT = PROJECT_ROOT / "data"
DATASETS = {
"skatingverse": {
"description": "28,579 RGB clips, 28 action classes (6 jump types x 4 rotations + 4 spins + NONE)",
"use": "primary training + evaluation",
"size": "~15 GB",
},
"mmfs": {
"description": "11,671 clips, 256 categories, skeleton data pre-extracted",
"note": "skeleton only; RGB requires email to liusl@dlut.edu.cn",
"use": "supplementary skeleton data + quality scores",
"size": "~2 GB skeleton",
},
"finefs": {
"description": "1,167 solo performances, 2-4 min each, temporal segmentation + scores",
"note": (
"videos (~46 GB) on Baidu Pan only; auto-downloaded via BaiduPCS-Go if "
"BAIDU_COOKIES/BAIDU_BDUSS is set, else falls back to manual instructions"
),
"use": "temporal boundary annotations + quality assessment labels",
"size": "~48 GB full / ~1 GB without videos",
},
"fs_jump3d": {
"description": "3D pose dataset for figure skating jumps, 12-view motion capture",
"use": "3D pose estimation validation + ground truth",
"size": "~9.6 GB full / ~808 MB without videos",
},
"fsd10": {
"description": (
"1,484 clips, 10 classes incl. a real named jump combo (Triple Lutz-Triple "
"Toeloop); skeleton only, no RGB (2017-2018 championship footage)"
),
"note": (
"Baidu Pan only, no smaller/alternate mirror found; auto-downloaded via "
"BaiduPCS-Go if BAIDU_COOKIES/BAIDU_BDUSS is set, else falls back to manual "
"instructions. Not to be confused with the different, unlabeled-by-name 30-class "
"'FSD' skeleton set mirrored on videotag.bj.bcebos.com from an unrelated 2021 "
"PaddlePaddle competition -- this is the original 10-class dataset with named "
"classes from the FSD-10 paper (Liu et al., Neurocomputing 2020)."
),
"use": "supplementary skeleton data with an explicit jump-combination class",
"size": "unconfirmed (not documented publicly)",
},
}
def _copy_tree_contents(src: Path, dst: Path, max_workers: int = 8) -> None:
"""Copy src's contents into dst, copying individual files in parallel.
Plain shutil.copytree copies one file at a time, which is dominated by
per-file syscall overhead when there are tens of thousands of small
clips. Copying is I/O-bound, so a thread pool lets multiple copies
overlap instead of waiting on each file in turn.
"""
dst.mkdir(parents=True, exist_ok=True)
for child in src.iterdir():
target = dst / child.name
if child.is_dir() and target.exists():
shutil.rmtree(target)
files = [(p, dst / p.relative_to(src)) for p in src.rglob("*") if p.is_file()]
for _, target in files:
target.parent.mkdir(parents=True, exist_ok=True)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
list(executor.map(lambda pair: shutil.copy2(*pair), files))
def _require_gdown():
try:
import gdown
return gdown
except ImportError as exc:
raise RuntimeError("Install gdown first: pip install gdown") from exc
def _download_gdrive_folder(folder_id: str, output_dir: Path) -> None:
gdown = _require_gdown()
output_dir.mkdir(parents=True, exist_ok=True)
url = f"https://drive.google.com/drive/folders/{folder_id}"
gdown.download_folder(url, output=str(output_dir), quiet=False, use_cookies=False)
def _download_gdrive_file(file_id: str, output_path: Path) -> None:
gdown = _require_gdown()
output_path.parent.mkdir(parents=True, exist_ok=True)
gdown.download(id=file_id, output=str(output_path), quiet=False)
_SKATINGVERSE_MODELSCOPE_NAMESPACE = "awei2003"
_SKATINGVERSE_MODELSCOPE_DATASET = "1st_SkatingVerse_Dataset"
_SKATINGVERSE_LABEL_FILES = ("train.txt", "mapping.txt", "answer.txt")
def _download_skatingverse_labels(output_dir: Path) -> None:
"""Fetch train.txt/mapping.txt/answer.txt from ModelScope's OSS-backed storage.
The Kaggle mirror only ships train_videos/ and test_videos/ with no label
file (parent-folder name is the only "label" and doesn't parse). The real
labels live in the dataset's ModelScope repo, but as OSS objects rather than
git-tracked files, so they're listed via the OSS tree API and fetched from
the presigned URLs it returns instead of a git clone.
"""
import requests
if all((output_dir / name).exists() for name in _SKATINGVERSE_LABEL_FILES):
print(" label files already present, skipping ModelScope fetch")
return
tree_url = (
f"https://www.modelscope.cn/api/v1/datasets/"
f"{_SKATINGVERSE_MODELSCOPE_NAMESPACE}/{_SKATINGVERSE_MODELSCOPE_DATASET}/oss/tree"
)
print(" fetching label files (train.txt, mapping.txt, answer.txt) from ModelScope")
resp = requests.get(tree_url, params={"MaxLimit": 200, "Revision": "master", "Recursive": "true"})
resp.raise_for_status()
entries = resp.json()["Data"]
by_name = {entry["Key"]: entry["Url"] for entry in entries}
missing = [name for name in _SKATINGVERSE_LABEL_FILES if name not in by_name]
if missing:
raise RuntimeError(f"ModelScope OSS listing is missing expected label file(s): {missing}")
for name in _SKATINGVERSE_LABEL_FILES:
dest = output_dir / name
if dest.exists():
continue
with requests.get(by_name[name], stream=True) as file_resp:
file_resp.raise_for_status()
with open(dest, "wb") as f:
for chunk in file_resp.iter_content(chunk_size=1 << 20):
f.write(chunk)
print(f" downloaded {name}")
def download_skatingverse(output_dir: Path, skip_videos: bool = False) -> Path:
"""Download SkatingVerse from Kaggle — actual RGB video clips.
Also fetches train.txt/mapping.txt/answer.txt from ModelScope, since the
Kaggle mirror doesn't include them (see _download_skatingverse_labels).
The ModelScope label fetch hits a different host and doesn't depend on
the Kaggle download, so it runs concurrently instead of after it.
"""
try:
import kagglehub
except ImportError as exc:
raise RuntimeError("Install kagglehub first: pip install kagglehub") from exc
with ThreadPoolExecutor(max_workers=2) as executor:
labels_future = executor.submit(_download_skatingverse_labels, output_dir)
downloaded = Path(kagglehub.dataset_download("elephantfish/skatingverse-dataset"))
_copy_tree_contents(downloaded, output_dir)
labels_future.result()
return output_dir
def download_mmfs(output_dir: Path, skip_videos: bool = False) -> Path:
"""Download MMFS skeleton data from Google Drive."""
gdown = _require_gdown()
output_dir.mkdir(parents=True, exist_ok=True)
gdrive_id = "12lhALM9u17-_V0pH8Epa7LgPJY4x3sM4"
url = f"https://drive.google.com/uc?id={gdrive_id}"
try:
gdown.download_folder(url, output=str(output_dir), quiet=False, use_cookies=False)
except Exception:
output_file = output_dir / f"{gdrive_id}.download"
gdown.download(id=gdrive_id, output=str(output_file), quiet=False)
return output_dir
_FINEFS_BAIDU_URL = "https://pan.baidu.com/s/1ihV47FIgNhATm5g1XTcaNg"
_FINEFS_BAIDU_CODE = "hri6"
_FINEFS_BAIDU_REMOTE_DIR = "/finefs_dataset"
_FINEFS_EXT_TO_SUBDIR = {
".mp4": "videos",
".npz": "skeletons",
".pkl": "features",
}
def _baidupcs_go_bin() -> str:
return os.environ.get("BAIDUPCS_GO_BIN", "BaiduPCS-Go")
def _run_baidupcs(*args: str) -> subprocess.CompletedProcess:
bin_path = _baidupcs_go_bin()
try:
return subprocess.run([bin_path, *args], capture_output=True, text=True)
except FileNotFoundError as exc:
raise RuntimeError(
"BaiduPCS-Go not found. Install a release binary from "
"https://github.com/qjfoidnh/BaiduPCS-Go/releases (or `go install "
"github.com/qjfoidnh/BaiduPCS-Go@latest`), or point BAIDUPCS_GO_BIN "
"at the binary."
) from exc
def _baidupcs_login() -> None:
"""Log BaiduPCS-Go into a real Baidu account so it can transfer + download at full speed.
Anonymous Baidu Pan downloads of large shared files are throttled to a few
KB/s, so this needs a real session: log into pan.baidu.com in a browser,
open devtools' Network tab, and copy the Cookie header of any
pan.baidu.com request into BAIDU_COOKIES (or extract BDUSS/STOKEN from it
into BAIDU_BDUSS/BAIDU_STOKEN). STOKEN is required for the transfer step.
"""
cookies = os.environ.get("BAIDU_COOKIES")
bduss = os.environ.get("BAIDU_BDUSS")
if cookies:
result = _run_baidupcs("login", f"-cookies={cookies}")
elif bduss:
args = ["login", f"-bduss={bduss}"]
stoken = os.environ.get("BAIDU_STOKEN")
if stoken:
args.append(f"-stoken={stoken}")
result = _run_baidupcs(*args)
else:
raise RuntimeError(
"Set BAIDU_COOKIES (or BAIDU_BDUSS[+BAIDU_STOKEN]) to a logged-in "
"pan.baidu.com session to download from Baidu Pan automatically."
)
if result.returncode != 0:
raise RuntimeError(f"BaiduPCS-Go login failed:\n{result.stdout}\n{result.stderr}")
def _sort_finefs_baidu_download(raw_dir: Path, output_dir: Path) -> None:
"""Route files pulled from the Baidu share into videos/skeletons/features by extension."""
for src in raw_dir.rglob("*"):
if not src.is_file():
continue
subdir = _FINEFS_EXT_TO_SUBDIR.get(src.suffix.lower())
if subdir is None:
continue
dst = output_dir / subdir / src.name
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dst))
shutil.rmtree(raw_dir, ignore_errors=True)
def _download_finefs_videos_via_baidupcs(output_dir: Path) -> None:
"""Transfer the FineFS Baidu share into our own account, then download it.
Baidu only lets shared files be downloaded at full speed after they're
"transferred" (saved) into a logged-in account's own drive -- anonymous
downloads of a multi-GB shared file are capped to a crawl. This requires
_baidupcs_login() to have set up a real session first.
"""
_run_baidupcs("mkdir", _FINEFS_BAIDU_REMOTE_DIR)
_run_baidupcs("cd", _FINEFS_BAIDU_REMOTE_DIR)
transfer = _run_baidupcs("transfer", _FINEFS_BAIDU_URL, _FINEFS_BAIDU_CODE, "-collect")
if transfer.returncode != 0:
raise RuntimeError(f"BaiduPCS-Go transfer failed:\n{transfer.stdout}\n{transfer.stderr}")
raw_dir = output_dir / "_baidu_download"
raw_dir.mkdir(parents=True, exist_ok=True)
download = _run_baidupcs("download", _FINEFS_BAIDU_REMOTE_DIR, f"--saveto={raw_dir}")
if download.returncode != 0:
raise RuntimeError(f"BaiduPCS-Go download failed:\n{download.stdout}\n{download.stderr}")
_sort_finefs_baidu_download(raw_dir, output_dir)
def download_finefs(output_dir: Path, skip_videos: bool = False) -> Path:
"""Download FineFS annotations + skeleton data, and attempt videos via Baidu Pan.
The repo itself only has documentation. The actual data lives on Baidu Pan:
https://pan.baidu.com/s/1ihV47FIgNhATm5g1XTcaNg (code: hri6)
The Google Drive link in the upstream README is broken (empty URL), and per
the maintainer (github.com/yanliji/FineFS-dataset issue #4), there is no
other mirror. Videos (~46 GB) are only available via Baidu Pan, whose
anonymous downloads are throttled hard, so this only succeeds with
BAIDU_COOKIES or BAIDU_BDUSS set (see _baidupcs_login docstring). If that
isn't set, or BaiduPCS-Go isn't installed, or the transfer/download itself
fails, this falls back to printing manual instructions instead of raising.
"""
repo_url = "https://github.com/yanliji/FineFS-dataset.git"
repo_dir = output_dir / "_repo"
if repo_dir.exists() and any(repo_dir.iterdir()):
subprocess.run(["git", "-C", str(repo_dir), "pull", "--ff-only"], check=True)
else:
repo_dir.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(["git", "clone", repo_url, str(repo_dir)], check=True)
annotations_dir = output_dir / "annotations"
for src in repo_dir.rglob("*.json"):
dst = annotations_dir / src.relative_to(repo_dir)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
for src in repo_dir.rglob("*.csv"):
dst = annotations_dir / src.relative_to(repo_dir)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
baidu_url = _FINEFS_BAIDU_URL
baidu_code = _FINEFS_BAIDU_CODE
if skip_videos:
print(" skipping videos (--skip-videos)")
else:
videos_dir = output_dir / "videos"
skeletons_dir = output_dir / "skeletons"
if videos_dir.exists() and any(videos_dir.iterdir()):
print(" videos directory already populated, skipping Baidu download")
else:
try:
print(" attempting automated Baidu Pan download via BaiduPCS-Go...")
_baidupcs_login()
_download_finefs_videos_via_baidupcs(output_dir)
print(f" downloaded videos/skeletons/features via BaiduPCS-Go -> {output_dir.resolve()}")
except Exception as exc:
print(f" automated download failed: {exc}")
print(f" WARNING: FineFS videos (~46 GB) must be downloaded manually from Baidu Pan.")
print(f" URL: {baidu_url}")
print(f" Code: {baidu_code}")
print(f" Place videos in: {videos_dir.resolve()}")
print(f" Place skeletons in: {skeletons_dir.resolve()}")
if not (output_dir / "videos").exists() or not any((output_dir / "videos").iterdir()):
readme = output_dir / "DOWNLOAD_INSTRUCTIONS.txt"
readme.write_text(
f"FineFS Dataset\n"
f"==============\n\n"
f"Annotations were extracted from the GitHub repo.\n\n"
f"Videos (~46 GB), skeletons (~828 MB), and features (~955 MB)\n"
f"live on Baidu Pan:\n\n"
f" URL: {baidu_url}\n"
f" Extraction code: {baidu_code}\n\n"
f"Option 1 (automated): set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with\n"
f"BAIDU_STOKEN) to a logged-in pan.baidu.com session and re-run this download --\n"
f"it will transfer + fetch the share via BaiduPCS-Go automatically. STOKEN is\n"
f"required for the transfer step; without it, the login succeeds but transfer\n"
f"will fail. See _baidupcs_login() in download.py for how to get these values.\n"
f"Requires BaiduPCS-Go: https://github.com/qjfoidnh/BaiduPCS-Go\n\n"
f"Option 2 (manual): download the share above by hand and place contents into\n"
f"this directory:\n"
f" videos/ -> .mp4 files\n"
f" skeletons/ -> .npz files\n"
f" features/ -> .pkl files\n\n"
f"The Google Drive link in the upstream repo README is broken, and per the\n"
f"maintainer (github.com/yanliji/FineFS-dataset issue #4) there is no other\n"
f"mirror as of 2026.\n",
encoding="utf-8",
)
return output_dir
_FS_JUMP3D_GDRIVE = {
"c3d": {
"folder_id": "1Ki9dxLuo78XFnCun9LGwWFlzO-A0FxJT",
"description": "C3D motion capture files (302.6 MB)",
},
"json": {
"folder_id": "17gQJR-qzF_JTs8JZgwZc1wuRKvkwnwVj",
"description": "JSON 3D pose files (505.2 MB)",
},
"videos": {
"folder_id": "1yvZMmK4hvrvK5ykqzkr1d-yVmImz-NNJ",
"description": "12-viewpoint video recordings (8.84 GB)",
},
}
def download_fs_jump3d(output_dir: Path, skip_videos: bool = False) -> Path:
"""Download FS-Jump3D 3D pose data from Google Drive.
Clones the repo for utilities (format.py, visualization.ipynb), then
downloads the actual data (C3D, JSON, videos) from Google Drive.
"""
repo_url = "https://github.com/ryota-skating/FS-Jump3D.git"
repo_dir = output_dir / "_repo"
if repo_dir.exists() and any(repo_dir.iterdir()):
subprocess.run(["git", "-C", str(repo_dir), "pull", "--ff-only"], check=True)
else:
repo_dir.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(["git", "clone", repo_url, str(repo_dir)], check=True)
utils_dst = output_dir / "utils"
utils_src = repo_dir / "utils"
if utils_src.exists():
_copy_tree_contents(utils_src, utils_dst)
for key, info in _FS_JUMP3D_GDRIVE.items():
if key == "videos" and skip_videos:
print(f" skipping {info['description']} (--skip-videos)")
continue
dest = output_dir / key
if dest.exists() and any(dest.iterdir()):
print(f" {key}/ already populated, skipping")
continue
print(f" downloading {info['description']}")
_download_gdrive_folder(info["folder_id"], dest)
return output_dir
_FSD10_BAIDU_URL = "https://pan.baidu.com/s/1d8dIxesymX00n9xCIHnVNg"
_FSD10_BAIDU_CODE = "c4oc"
_FSD10_BAIDU_REMOTE_DIR = "/fsd10_dataset"
def download_fsd10(output_dir: Path, skip_videos: bool = False) -> Path:
"""Download FSD-10 (skeleton-only, no RGB) from Baidu Pan.
No GitHub repo or alternate mirror exists for this one (checked HuggingFace, ModelScope,
OpenDataLab, Kaggle, Academic Torrents -- nothing). skip_videos is accepted for interface
consistency with the other downloaders but has no effect: the dataset never had RGB video
to begin with ("RGB datasets would not be provided for copyright reasons" per the
maintainer's own PaddleVideo docs), so there's nothing to skip.
Do not confuse this with the different, unrelated 30-class "FSD" skeleton set mirrored on
videotag.bj.bcebos.com (from a 2021 PaddlePaddle/CCF competition) -- that one is a clean,
ungated direct download, but its 30 labels have no publicly documented class names, which
defeats the point of wanting FSD-10's real named classes (e.g. its combination-jump class).
"""
output_dir.mkdir(parents=True, exist_ok=True)
try:
print(" attempting automated Baidu Pan download via BaiduPCS-Go...")
_baidupcs_login()
_run_baidupcs("mkdir", _FSD10_BAIDU_REMOTE_DIR)
_run_baidupcs("cd", _FSD10_BAIDU_REMOTE_DIR)
transfer = _run_baidupcs("transfer", _FSD10_BAIDU_URL, _FSD10_BAIDU_CODE, "-collect")
if transfer.returncode != 0:
raise RuntimeError(f"BaiduPCS-Go transfer failed:\n{transfer.stdout}\n{transfer.stderr}")
download = _run_baidupcs("download", _FSD10_BAIDU_REMOTE_DIR, f"--saveto={output_dir}")
if download.returncode != 0:
raise RuntimeError(f"BaiduPCS-Go download failed:\n{download.stdout}\n{download.stderr}")
print(f" downloaded FSD-10 via BaiduPCS-Go -> {output_dir.resolve()}")
except Exception as exc:
print(f" automated download failed: {exc}")
print(" WARNING: FSD-10 must be downloaded manually from Baidu Pan.")
print(f" URL: {_FSD10_BAIDU_URL}")
print(f" Code: {_FSD10_BAIDU_CODE}")
print(f" Place downloaded contents into: {output_dir.resolve()}")
readme = output_dir / "DOWNLOAD_INSTRUCTIONS.txt"
readme.write_text(
f"FSD-10 Dataset\n"
f"==============\n\n"
f"Skeleton-only (OpenPose-derived), 10 named classes incl. a real jump-combination\n"
f"class (Triple Lutz-Triple Toeloop). No RGB video is provided by the maintainers.\n\n"
f"Only available via Baidu Pan (no other mirror found):\n\n"
f" URL: {_FSD10_BAIDU_URL}\n"
f" Extraction code: {_FSD10_BAIDU_CODE}\n\n"
f"Option 1 (automated): set BAIDU_COOKIES (or BAIDU_BDUSS, optionally with\n"
f"BAIDU_STOKEN) to a logged-in pan.baidu.com session and re-run this download --\n"
f"it will transfer + fetch the share via BaiduPCS-Go automatically.\n"
f"Requires BaiduPCS-Go: https://github.com/qjfoidnh/BaiduPCS-Go\n\n"
f"Option 2 (manual): download the share above by hand into this directory.\n\n"
f"Contact for questions: liusl@mail.dlut.edu.cn\n",
encoding="utf-8",
)
return output_dir
DOWNLOADERS = {
"skatingverse": download_skatingverse,
"mmfs": download_mmfs,
"finefs": download_finefs,
"fsd10": download_fsd10,
"fs_jump3d": download_fs_jump3d,
}
def download_dataset(name: str, data_root: Path = DATA_ROOT, skip_videos: bool = False) -> Path:
"""Download one configured dataset into data/{name}/."""
if name not in DATASETS:
raise ValueError(f"Unknown dataset {name!r}. Choose from: {', '.join(DATASETS)}")
output_dir = data_root / name
output_dir.mkdir(parents=True, exist_ok=True)
return DOWNLOADERS[name](output_dir, skip_videos=skip_videos)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--all", action="store_true", help="Download all configured datasets")
group.add_argument("--only", nargs="+", choices=sorted(DATASETS), help="Download selected datasets")
parser.add_argument("--data-root", type=Path, default=DATA_ROOT, help="Output data root")
parser.add_argument(
"--skip-videos", action="store_true",
help="Skip large video downloads (annotations and skeleton data only)",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
selected = sorted(DATASETS) if args.all else args.only
print(f"Downloading {len(selected)} dataset(s) into {args.data_root.resolve()}")
if args.skip_videos:
print("Video downloads will be skipped (--skip-videos)")
successes = []
failures = []
for name in selected:
dataset = DATASETS[name]
print(f"\n{'=' * 60}")
print(f"{name}: {dataset['description']}")
print(f" use: {dataset['use']}")
print(f" size: {dataset['size']}")
if "note" in dataset:
print(f" note: {dataset['note']}")
try:
output = download_dataset(name, args.data_root, skip_videos=args.skip_videos)
successes.append((name, output))
print(f" -> {output.resolve()}")
except Exception as exc:
failures.append((name, exc))
print(f" FAILED: {exc}")
print(f"\n{'=' * 60}")
print("Summary")
for name, output in successes:
print(f" ok {name}: {output.resolve()}")
for name, exc in failures:
print(f" FAILED {name}: {exc}")
return 1 if failures else 0
if __name__ == "__main__":
raise SystemExit(main())

Xet Storage Details

Size:
24 kB
·
Xet hash:
cff13dc799b4edd42c281fa625112cd421eee5a0073087ddab5733ec47feb381

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.