push_to_jobs: --replace-extracted; cache/pip/tmp on bucket

.gitignore

@@ -1,17 +1,17 @@
-.venv/
-venv/
-data/
-__pycache__/
-*.pyc
-*.pyo
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-*.egg-info/
-dist/
-build/
-.git/
-.cursor/
-agent-tools/
-*.pt
-.DS_Store
+.venv/
+venv/
+data/
+__pycache__/
+*.pyc
+*.pyo
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+*.egg-info/
+dist/
+build/
+.git/
+.cursor/
+agent-tools/
+*.pt
+.DS_Store

configs/jobs_overrides.yaml

@@ -1,20 +1,20 @@
-# HF Jobs 训练覆盖项（与 configs/default.yaml 深度合并）。
-# 用法: python -m wjad.train.runner_local --config configs/default.yaml --config_overrides configs/jobs_overrides.yaml
-
-train:
-  batch_size: 12
-  grad_accum_steps: 1   # 显存不够可改大累积步并减小 batch
-
-data:
-  use_synthetic: true
-  use_real: true
-
-deploy:
-  hf_code_repo: "fuzirui/WJAD"
-  hf_weights_repo: "fuzirui/WJAD"
-  hf_bucket_id: "fuzirui/WJAD"
-  mirror_src_uri: "hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/"
-  mirror_bucket_subpath: "mirror/cosmos_hub"
-  extracted_bucket_subpath: "extracted/cosmos_hub"
-  cache_bucket_subpath: "cache"
-  runs_bucket_subpath: "runs"
+# HF Jobs 训练覆盖项（与 configs/default.yaml 深度合并）。
+# 用法: python -m wjad.train.runner_local --config configs/default.yaml --config_overrides configs/jobs_overrides.yaml
+
+train:
+  batch_size: 12
+  grad_accum_steps: 1   # 显存不够可改大累积步并减小 batch
+
+data:
+  use_synthetic: true
+  use_real: true
+
+deploy:
+  hf_code_repo: "fuzirui/WJAD"
+  hf_weights_repo: "fuzirui/WJAD"
+  hf_bucket_id: "fuzirui/WJAD"
+  mirror_src_uri: "hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/"
+  mirror_bucket_subpath: "mirror/cosmos_hub"
+  extracted_bucket_subpath: "extracted/cosmos_hub"
+  cache_bucket_subpath: "cache"
+  runs_bucket_subpath: "runs"

scripts/clear_wjad_extract_state.py

@@ -0,0 +1,73 @@
+"""删除 ``jobs_extract_archives.py`` 在 ``--out-root`` 下生成的 ``_wjad_extract_state`` 目录。
+
+只删「已解压」进度标记，**不会**删除 ``extracted/`` 里真正的数据集文件。
+删完后可重新跑解压脚本，会按归档从头再解压一遍（若目录已存在可能覆盖/混合，按需先清 extracted）。
+
+用法示例::
+
+    # 本机 / Bucket 挂载路径与解压时 --out-root 一致
+    python scripts/clear_wjad_extract_state.py --out-root /mnt/wjad/extracted/cosmos_hub
+
+    # 只看会删什么，不真删
+    python scripts/clear_wjad_extract_state.py --out-root F:/bucket/extracted/cosmos_hub --dry-run
+
+环境变量（可选）::
+
+    设置 WJAD_EXTRACTED_ROOT 后可省略 --out-root
+
+    Linux Job::
+
+        export WJAD_EXTRACTED_ROOT=/mnt/wjad/extracted/cosmos_hub
+        python scripts/clear_wjad_extract_state.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import sys
+from pathlib import Path
+
+STATE_DIRNAME = "_wjad_extract_state"
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(description="Remove _wjad_extract_state under extracted out-root.")
+    p.add_argument(
+        "--out-root",
+        type=Path,
+        default=None,
+        help="与 jobs_extract_archives --out-root 相同；不设则用环境变量 WJAD_EXTRACTED_ROOT",
+    )
+    p.add_argument("--dry-run", action="store_true", help="只打印路径，不删除")
+    args = p.parse_args()
+    out = args.out_root
+    if out is None:
+        env = os.environ.get("WJAD_EXTRACTED_ROOT")
+        if not env:
+            print(
+                "需要 --out-root 或环境变量 WJAD_EXTRACTED_ROOT",
+                file=sys.stderr,
+            )
+            sys.exit(2)
+        out = Path(env)
+    out = out.resolve()
+    state = out / STATE_DIRNAME
+    if not state.exists():
+        print(f"[clear] 不存在，跳过: {state}")
+        return
+    if not state.is_dir():
+        print(f"[clear] 不是目录，拒绝: {state}", file=sys.stderr)
+        sys.exit(1)
+    print(f"[clear] {'(dry-run) ' if args.dry_run else ''}目标: {state}")
+    if args.dry_run:
+        n = sum(1 for _ in state.rglob("*"))
+        print(f"[clear] dry-run: 其下约 {n} 个条目（含文件与目录）")
+        return
+    shutil.rmtree(state)
+    print(f"[clear] 已删除: {state}")
+
+
+if __name__ == "__main__":
+    main()

scripts/jobs_entry_train.sh

@@ -1,51 +1,57 @@
-#!/usr/bin/env bash
-# HF Job（GPU）入口：在仓库根目录执行（由 push_to_jobs 先 clone 再调用本脚本）。
-# 数据与缓存路径默认指向已挂载的 Bucket（WJAD_BUCKET_MOUNT），避免占用 Job 本地盘。
-set -euo pipefail
-
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$REPO_ROOT"
-
-BUCKET="${WJAD_BUCKET_MOUNT:-/mnt/wjad}"
-HUB_REPO="${WJAD_HUB_REPO:-fuzirui/WJAD}"
-# 缓存统一在 bucket 的 cache/ 下，与镜像(mirror)和解压数据(extracted)分离，可整目录删除清缓存
-CACHE_ROOT="${WJAD_CACHE_ROOT:-${BUCKET}/cache}"
-
-export HF_HOME="${CACHE_ROOT}/hf"
-export TRANSFORMERS_CACHE="${CACHE_ROOT}/transformers"
-export TORCH_HOME="${CACHE_ROOT}/torch"
-export HF_DATASETS_CACHE="${CACHE_ROOT}/datasets"
-export XDG_CACHE_HOME="${CACHE_ROOT}/xdg"
-mkdir -p "${HF_HOME}" "${TRANSFORMERS_CACHE}" "${TORCH_HOME}" "${HF_DATASETS_CACHE}" "${XDG_CACHE_HOME}"
-
-if command -v apt-get >/dev/null 2>&1; then
-  apt-get update && apt-get install -y --no-install-recommends git ffmpeg libgl1 libglib2.0-0 || true
-fi
-
-pip install -q -U pip huggingface_hub
-pip install -q -e .
-
-export WJAD_OUTPUT_DIR="${WJAD_OUTPUT_DIR:-${BUCKET}/runs/current}"
-export WJAD_HUB_REPO="${HUB_REPO}"
-# 训练只读解压产物，不与 mirror 混用（解压脚本写入 extracted/，保持与源相同的相对路径树）
-export WJAD_DATA_ROOT="${WJAD_DATA_ROOT:-${BUCKET}/extracted/cosmos_hub}"
-mkdir -p "${WJAD_OUTPUT_DIR}"
-
-read -r -a BS_TRY <<< "${WJAD_OVERRIDE_BS_LIST:-12 10 8 6 4 2}"
-for BS in "${BS_TRY[@]}"; do
-  echo "[jobs_entry_train] try batch_size=${BS}" >&2
-  export WJAD_BATCH_SIZE="${BS}"
-  if python -m wjad.train.runner_local \
-      --config configs/default.yaml \
-      --config_overrides configs/jobs_overrides.yaml \
-      --device cuda \
-      --data_root "${WJAD_DATA_ROOT}" \
-      --dinov3_path ./dinov3-vitb16-pretrain-lvd1689m \
-      --output_dir "${WJAD_OUTPUT_DIR}" \
-      --hub_repo "${HUB_REPO}"; then
-    echo "[jobs_entry_train] finished @ BS=${BS}" >&2
-    exit 0
-  fi
-  echo "[jobs_entry_train] run failed @ BS=${BS}, lowering batch" >&2
-done
-exit 1
+#!/usr/bin/env bash
+# HF Job（GPU）入口：在仓库根目录执行（由 push_to_jobs 先 clone 再调用本脚本）。
+# 数据与缓存路径默认指向已挂载的 Bucket（WJAD_BUCKET_MOUNT），避免占用 Job 本地盘。
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_ROOT"
+
+BUCKET="${WJAD_BUCKET_MOUNT:-/mnt/wjad}"
+HUB_REPO="${WJAD_HUB_REPO:-fuzirui/WJAD}"
+# 缓存统一在 bucket 的 cache/ 下，与镜像(mirror)和解压数据(extracted)分离，可整目录删除清缓存
+CACHE_ROOT="${WJAD_CACHE_ROOT:-${BUCKET}/cache}"
+
+export HF_HOME="${CACHE_ROOT}/hf"
+export TRANSFORMERS_CACHE="${CACHE_ROOT}/transformers"
+export TORCH_HOME="${CACHE_ROOT}/torch"
+export HF_DATASETS_CACHE="${CACHE_ROOT}/datasets"
+export XDG_CACHE_HOME="${CACHE_ROOT}/xdg"
+# pip / 临时文件也进 Bucket，避免 K8s ephemeral 50G 限额被占满
+export PIP_CACHE_DIR="${CACHE_ROOT}/pip"
+export TMPDIR="${CACHE_ROOT}/tmp"
+export TEMP="${TMPDIR}"
+export TMP="${TMPDIR}"
+mkdir -p "${HF_HOME}" "${TRANSFORMERS_CACHE}" "${TORCH_HOME}" "${HF_DATASETS_CACHE}" "${XDG_CACHE_HOME}" \
+  "${PIP_CACHE_DIR}" "${TMPDIR}"
+
+if command -v apt-get >/dev/null 2>&1; then
+  apt-get update && apt-get install -y --no-install-recommends git ffmpeg libgl1 libglib2.0-0 || true
+fi
+
+pip install -q -U pip huggingface_hub
+pip install -q -e .
+
+export WJAD_OUTPUT_DIR="${WJAD_OUTPUT_DIR:-${BUCKET}/runs/current}"
+export WJAD_HUB_REPO="${HUB_REPO}"
+# 训练只读解压产物，不与 mirror 混用（解压脚本写入 extracted/，保持与源相同的相对路径树）
+export WJAD_DATA_ROOT="${WJAD_DATA_ROOT:-${BUCKET}/extracted/cosmos_hub}"
+mkdir -p "${WJAD_OUTPUT_DIR}"
+
+read -r -a BS_TRY <<< "${WJAD_OVERRIDE_BS_LIST:-12 10 8 6 4 2}"
+for BS in "${BS_TRY[@]}"; do
+  echo "[jobs_entry_train] try batch_size=${BS}" >&2
+  export WJAD_BATCH_SIZE="${BS}"
+  if python -m wjad.train.runner_local \
+      --config configs/default.yaml \
+      --config_overrides configs/jobs_overrides.yaml \
+      --device cuda \
+      --data_root "${WJAD_DATA_ROOT}" \
+      --dinov3_path ./dinov3-vitb16-pretrain-lvd1689m \
+      --output_dir "${WJAD_OUTPUT_DIR}" \
+      --hub_repo "${HUB_REPO}"; then
+    echo "[jobs_entry_train] finished @ BS=${BS}" >&2
+    exit 0
+  fi
+  echo "[jobs_entry_train] run failed @ BS=${BS}, lowering batch" >&2
+done
+exit 1

scripts/jobs_extract_archives.py

@@ -1,119 +1,119 @@
-"""在已挂载的 Bucket 上解压归档：写入单独的 ``extracted/`` 树，不与 ``mirror/`` 混放。
-
-扫描 ``--scan-root``（通常为 ``mirror/cosmos_hub``，即 Hub copy 镜像）下的归档；
-解压到 ``--out-root`` / (相对 scan-root 的父路径) / (归档主文件名)/ ，
-从而在 ``extracted/`` 下复现与源数据相同的**相对目录结构**，避免与镜像目录混淆。
-
-解压产物只写在 ``--out-root`` 下；**不在 mirror 里落任何文件**（进度标记也放在
-``out-root/_wjad_extract_state/``，与归档相对路径对应，避免修改 ``--scan-root``）。
-
-示例（mount ``hf://buckets/.../WJAD`` → ``/mnt/wjad``）::
-
-    python scripts/jobs_extract_archives.py \\
-        --scan-root /mnt/wjad/mirror/cosmos_hub \\
-        --out-root /mnt/wjad/extracted/cosmos_hub
-"""
-
-from __future__ import annotations
-
-import argparse
-import tarfile
-import zipfile
-from pathlib import Path
-
-
-def _archive_stem(path: Path) -> str:
-    n = path.name
-    lower = n.lower()
-    for ext in (".tar.gz", ".tar.bz2", ".tgz"):
-        if lower.endswith(ext):
-            return n[: -len(ext)]
-    if lower.endswith(".tar"):
-        return n[:-4]
-    if lower.endswith(".zip"):
-        return n[:-4]
-    return path.stem
-
-
-def _is_archive(path: Path) -> bool:
-    lower = path.name.lower()
-    return lower.endswith(
-        (".tar.gz", ".tar.bz2", ".tgz", ".tar.xz", ".tar", ".zip"),
-    )
-
-
-STATE_DIRNAME = "_wjad_extract_state"
-
-
-def _done_marker_path(archive: Path, scan: Path, out_root: Path) -> Path:
-    """标记只写在 out_root 下，绝不写回 mirror。"""
-    rel = archive.relative_to(scan)
-    return out_root / STATE_DIRNAME / rel.parent / (rel.name + ".wjad_done")
-
-
-def _validate_roots(scan: Path, out_root: Path) -> None:
-    s, o = scan.resolve(), out_root.resolve()
-    if s == o:
-        raise SystemExit("--out-root 不能与 --scan-root 相同，否则会写回镜像目录")
-    try:
-        o.relative_to(s)
-        raise SystemExit("--out-root 不能位于 --scan-root 内部（mirror 只读，解压请用 extracted/）")
-    except ValueError:
-        pass
-
-
-def _extract_one(archive: Path, dest_dir: Path) -> None:
-    dest_dir.mkdir(parents=True, exist_ok=True)
-    lower = archive.name.lower()
-    if lower.endswith(".zip"):
-        with zipfile.ZipFile(archive, "r") as z:
-            z.extractall(dest_dir)
-        return
-    mode = "r"
-    if lower.endswith((".tar.gz", ".tgz")):
-        mode = "r:gz"
-    elif lower.endswith(".tar.bz2"):
-        mode = "r:bz2"
-    elif lower.endswith(".tar.xz"):
-        mode = "r:xz"
-    elif lower.endswith(".tar"):
-        mode = "r:"
-    else:
-        raise ValueError(f"unsupported archive: {archive}")
-    with tarfile.open(archive, mode) as tf:
-        try:
-            tf.extractall(dest_dir, filter=tarfile.data_filter)  # py3.12+
-        except TypeError:
-            tf.extractall(dest_dir)
-
-
-def main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument("--scan-root", type=Path, required=True)
-    p.add_argument("--out-root", type=Path, required=True)
-    args = p.parse_args()
-    scan: Path = args.scan_root.resolve()
-    out_root: Path = args.out_root.resolve()
-    if not scan.is_dir():
-        raise SystemExit(f"--scan-root not a directory: {scan}")
-    _validate_roots(scan, out_root)
-
-    count = 0
-    for path in sorted(scan.rglob("*")):
-        if not path.is_file() or not _is_archive(path):
-            continue
-        mpath = _done_marker_path(path, scan, out_root)
-        if mpath.exists():
-            continue
-        rel_parent = path.parent.relative_to(scan)
-        dest = out_root / rel_parent / _archive_stem(path)
-        print(f"[extract] {path.relative_to(scan)} -> {dest.relative_to(out_root)}", flush=True)
-        _extract_one(path, dest)
-        mpath.parent.mkdir(parents=True, exist_ok=True)
-        mpath.write_text("ok\n", encoding="utf-8")
-        count += 1
-    print(f"[extract] done, {count} archives", flush=True)
-
-
-if __name__ == "__main__":
-    main()
+"""在已挂载的 Bucket 上解压归档：写入单独的 ``extracted/`` 树，不与 ``mirror/`` 混放。
+
+扫描 ``--scan-root``（通常为 ``mirror/cosmos_hub``，即 Hub copy 镜像）下的归档；
+解压到 ``--out-root`` / (相对 scan-root 的父路径) / (归档主文件名)/ ，
+从而在 ``extracted/`` 下复现与源数据相同的**相对目录结构**，避免与镜像目录混淆。
+
+解压产物只写在 ``--out-root`` 下；**不在 mirror 里落任何文件**（进度标记也放在
+``out-root/_wjad_extract_state/``，与归档相对路径对应，避免修改 ``--scan-root``）。
+
+示例（mount ``hf://buckets/.../WJAD`` → ``/mnt/wjad``）::
+
+    python scripts/jobs_extract_archives.py \\
+        --scan-root /mnt/wjad/mirror/cosmos_hub \\
+        --out-root /mnt/wjad/extracted/cosmos_hub
+"""
+
+from __future__ import annotations
+
+import argparse
+import tarfile
+import zipfile
+from pathlib import Path
+
+
+def _archive_stem(path: Path) -> str:
+    n = path.name
+    lower = n.lower()
+    for ext in (".tar.gz", ".tar.bz2", ".tgz"):
+        if lower.endswith(ext):
+            return n[: -len(ext)]
+    if lower.endswith(".tar"):
+        return n[:-4]
+    if lower.endswith(".zip"):
+        return n[:-4]
+    return path.stem
+
+
+def _is_archive(path: Path) -> bool:
+    lower = path.name.lower()
+    return lower.endswith(
+        (".tar.gz", ".tar.bz2", ".tgz", ".tar.xz", ".tar", ".zip"),
+    )
+
+
+STATE_DIRNAME = "_wjad_extract_state"
+
+
+def _done_marker_path(archive: Path, scan: Path, out_root: Path) -> Path:
+    """标记只写在 out_root 下，绝不写回 mirror。"""
+    rel = archive.relative_to(scan)
+    return out_root / STATE_DIRNAME / rel.parent / (rel.name + ".wjad_done")
+
+
+def _validate_roots(scan: Path, out_root: Path) -> None:
+    s, o = scan.resolve(), out_root.resolve()
+    if s == o:
+        raise SystemExit("--out-root 不能与 --scan-root 相同，否则会写回镜像目录")
+    try:
+        o.relative_to(s)
+        raise SystemExit("--out-root 不能位于 --scan-root 内部（mirror 只读，解压请用 extracted/）")
+    except ValueError:
+        pass
+
+
+def _extract_one(archive: Path, dest_dir: Path) -> None:
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    lower = archive.name.lower()
+    if lower.endswith(".zip"):
+        with zipfile.ZipFile(archive, "r") as z:
+            z.extractall(dest_dir)
+        return
+    mode = "r"
+    if lower.endswith((".tar.gz", ".tgz")):
+        mode = "r:gz"
+    elif lower.endswith(".tar.bz2"):
+        mode = "r:bz2"
+    elif lower.endswith(".tar.xz"):
+        mode = "r:xz"
+    elif lower.endswith(".tar"):
+        mode = "r:"
+    else:
+        raise ValueError(f"unsupported archive: {archive}")
+    with tarfile.open(archive, mode) as tf:
+        try:
+            tf.extractall(dest_dir, filter=tarfile.data_filter)  # py3.12+
+        except TypeError:
+            tf.extractall(dest_dir)
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--scan-root", type=Path, required=True)
+    p.add_argument("--out-root", type=Path, required=True)
+    args = p.parse_args()
+    scan: Path = args.scan_root.resolve()
+    out_root: Path = args.out_root.resolve()
+    if not scan.is_dir():
+        raise SystemExit(f"--scan-root not a directory: {scan}")
+    _validate_roots(scan, out_root)
+
+    count = 0
+    for path in sorted(scan.rglob("*")):
+        if not path.is_file() or not _is_archive(path):
+            continue
+        mpath = _done_marker_path(path, scan, out_root)
+        if mpath.exists():
+            continue
+        rel_parent = path.parent.relative_to(scan)
+        dest = out_root / rel_parent / _archive_stem(path)
+        print(f"[extract] {path.relative_to(scan)} -> {dest.relative_to(out_root)}", flush=True)
+        _extract_one(path, dest)
+        mpath.parent.mkdir(parents=True, exist_ok=True)
+        mpath.write_text("ok\n", encoding="utf-8")
+        count += 1
+    print(f"[extract] done, {count} archives", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/jobs_hub_copy_to_bucket.py

@@ -1,31 +1,31 @@
-"""Hub 服务端 copy：把 Hub 上已有 dataset/model/space 树拷贝到 Bucket（大文件走 xet hash，不占 Job 本地带宽）。
-
-示例（在 HF Job 内，仅需 HF_TOKEN）::
-
-    python scripts/jobs_hub_copy_to_bucket.py \\
-        --src hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/ \\
-        --dst hf://buckets/fuzirui/WJAD/mirror/cosmos_hub/
-"""
-
-from __future__ import annotations
-
-import argparse
-
-from huggingface_hub import copy_files
-
-
-def main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument("--src", required=True, help="hf://datasets/... 或 hf://user/model/…")
-    p.add_argument(
-        "--dst",
-        required=True,
-        help="目标必须为 bucket，如 hf://buckets/fuzirui/WJAD/mirror/cosmos_hub/",
-    )
-    args = p.parse_args()
-    copy_files(args.src, args.dst)
-    print("[copy] OK", args.src, "->", args.dst)
-
-
-if __name__ == "__main__":
-    main()
+"""Hub 服务端 copy：把 Hub 上已有 dataset/model/space 树拷贝到 Bucket（大文件走 xet hash，不占 Job 本地带宽）。
+
+示例（在 HF Job 内，仅需 HF_TOKEN）::
+
+    python scripts/jobs_hub_copy_to_bucket.py \\
+        --src hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/ \\
+        --dst hf://buckets/fuzirui/WJAD/mirror/cosmos_hub/
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from huggingface_hub import copy_files
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--src", required=True, help="hf://datasets/... 或 hf://user/model/…")
+    p.add_argument(
+        "--dst",
+        required=True,
+        help="目标必须为 bucket，如 hf://buckets/fuzirui/WJAD/mirror/cosmos_hub/",
+    )
+    args = p.parse_args()
+    copy_files(args.src, args.dst)
+    print("[copy] OK", args.src, "->", args.dst)
+
+
+if __name__ == "__main__":
+    main()

scripts/push_to_jobs.py

@@ -1,306 +1,339 @@
-"""提交 Hugging Face Jobs：数据集服务端 copy → CPU 挂载解压 → A10G-Large 正式训练。
-
-**持久化只在 Bucket 上**（挂载如 ``/mnt/wjad``）。Job 容器本地仅存 ``git clone`` 与 pip
-临时文件，大数据放不进本地盘。
-
-Bucket 目录约定（相对挂载根）::
-
-- ``mirror/cosmos_hub/`` — ``copy_files`` 得到的 Hub 数据集镜像；**只作解压源**，不把解压产物写回这里。
-- ``extracted/cosmos_hub/`` — **仅解压输出**，相对路径与源一致，训练 ``--data_root`` 指向这里。
-- ``cache/`` — 所有 HF / PyTorch / transformers 缓存根目录，**可整目录删除** 以清缓存。
-- ``runs/current/`` — checkpoint（仍在 Bucket）；另可 ``upload_file`` 到 Hub model repo。
-
-常用命令::
-
-    python scripts/push_to_jobs.py copy-extract   # 先 copy（等待成功）再 submit extract（前台跑完日志）
-
-单独步骤加 ``--detach``。Windows 上会使用 ``sys.executable`` 同目录的 ``hf.exe``。
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import shlex
-import shutil
-import subprocess
-import sys
-import time
-from pathlib import Path
-
-DEFAULT_BUCKET = "fuzirui/WJAD"
-DEFAULT_CODE = "fuzirui/WJAD"
-DEFAULT_WEIGHTS = "fuzirui/WJAD"
-DEFAULT_SRC = "hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/"
-DEFAULT_MIRROR = "mirror/cosmos_hub"
-DEFAULT_EXTRACTED = "extracted/cosmos_hub"
-DEFAULT_CACHE = "cache"
-DEFAULT_TIMEOUT = "7d"
-TRAIN_IMAGE = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime"
-TRAIN_FLAVOR = "a10g-large"
-
-
-def _hf_executable() -> str:
-    sibling = Path(sys.executable).parent / ("hf.exe" if sys.platform == "win32" else "hf")
-    if sibling.is_file():
-        return str(sibling.resolve())
-    w = shutil.which("hf")
-    return w if w else "hf"
-
-
-def _hf_jobs_cmd(parts: list[str]) -> list[str]:
-    if parts and parts[0] == "hf":
-        return [_hf_executable(), *parts[1:]]
-    return parts
-
-
-def _insert_detach(cmd: list[str], detach: bool) -> None:
-    if not detach:
-        return
-    idx = cmd.index("run") + 1
-    cmd.insert(idx, "--detach")
-
-
-def build_copy_cmd(args: argparse.Namespace) -> list[str]:
-    dst = f"hf://buckets/{args.bucket}/{args.mirror_subpath.rstrip('/')}/"
-    one_py = (
-        "import subprocess,sys;"
-        "subprocess.check_call([sys.executable,'-m','pip','install','-q','-U','huggingface_hub']);"
-        "from huggingface_hub import copy_files;"
-        f"copy_files({args.src!r},{dst!r})"
-    )
-    cmd = [
-        "hf",
-        "jobs",
-        "run",
-        "--flavor",
-        "cpu-basic",
-        "--timeout",
-        args.timeout,
-        "--secrets",
-        "HF_TOKEN",
-        "python:3.12",
-        "python",
-        "-c",
-        one_py,
-    ]
-    _insert_detach(cmd, args.detach)
-    return cmd
-
-
-def build_extract_cmd(args: argparse.Namespace) -> list[str]:
-    vol = f"hf://buckets/{args.bucket}:/mnt/wjad"
-    inner = " && ".join(
-        [
-            "set -e",
-            "command -v git >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends git)",
-            "pip install -q -U huggingface_hub",
-            f"git clone https://oauth2:$HF_TOKEN@huggingface.co/{args.code_repo} /tmp/wjad",
-            "cd /tmp/wjad",
-            "python scripts/jobs_extract_archives.py "
-            f"--scan-root /mnt/wjad/{args.mirror_subpath} --out-root /mnt/wjad/{args.extracted_subpath}",
-        ]
-    )
-    cmd = [
-        "hf",
-        "jobs",
-        "run",
-        "-v",
-        vol,
-        "--flavor",
-        "cpu-basic",
-        "--timeout",
-        args.timeout,
-        "--secrets",
-        "HF_TOKEN",
-        "python:3.12",
-        "sh",
-        "-c",
-        inner,
-    ]
-    _insert_detach(cmd, args.detach)
-    return cmd
-
-
-def build_train_cmd(args: argparse.Namespace) -> list[str]:
-    vol = f"hf://buckets/{args.bucket}:/mnt/wjad"
-    inner = " && ".join(
-        [
-            "set -e",
-            "export WJAD_BUCKET_MOUNT=/mnt/wjad",
-            f"export WJAD_CACHE_ROOT=/mnt/wjad/{args.cache_subpath}",
-            f"export WJAD_HUB_REPO={args.weights_repo}",
-            f"export WJAD_DATA_ROOT=/mnt/wjad/{args.extracted_subpath}",
-            "export WJAD_OUTPUT_DIR=/mnt/wjad/runs/current",
-            "command -v git >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends git)",
-            "pip install -q -U huggingface_hub",
-            f"git clone https://oauth2:$HF_TOKEN@huggingface.co/{args.code_repo} /workspace/wjad",
-            "cd /workspace/wjad",
-            "pip install -q -U pip",
-            "pip install -q -e .",
-            "bash scripts/jobs_entry_train.sh",
-        ]
-    )
-    cmd = [
-        "hf",
-        "jobs",
-        "run",
-        "-v",
-        vol,
-        "--flavor",
-        args.train_flavor,
-        "--timeout",
-        args.timeout,
-        "--secrets",
-        "HF_TOKEN",
-        args.train_image,
-        "bash",
-        "-lc",
-        inner,
-    ]
-    _insert_detach(cmd, args.detach)
-    return cmd
-
-
-def _run(cmd: list[str]) -> int:
-    print("[push_to_jobs] $", " ".join(shlex.quote(c) for c in cmd))
-    cmd = _hf_jobs_cmd(cmd)
-    return subprocess.call(cmd)
-
-
-def _run_capture(cmd: list[str]) -> subprocess.CompletedProcess:
-    cmd = _hf_jobs_cmd(cmd)
-    print("[push_to_jobs] $", " ".join(shlex.quote(c) for c in cmd))
-    return subprocess.run(cmd, capture_output=True, text=True)
-
-
-def _parse_detach_job_id(stdout: str, stderr: str) -> str | None:
-    text = (stdout + "\n" + stderr).strip()
-    m = re.search(r"\bID:\s*([a-fA-F0-9]{12,})\b", text)
-    if m:
-        return m.group(1)
-    m = re.search(r"/jobs/[^/\s]+/([a-fA-F0-9]{12,})", text)
-    if m:
-        return m.group(1)
-    for line in text.splitlines():
-        line = line.strip()
-        low = line.lower()
-        if "job" in low and "id" in low:
-            parts = line.replace(":", " ").split()
-            for i, p in enumerate(parts):
-                if p.lower() == "id" and i + 1 < len(parts):
-                    return parts[i + 1].strip().rstrip(",")
-    return None
-
-
-def _job_status(job_id: str) -> str | None:
-    r = _run_capture(["hf", "jobs", "inspect", job_id, "--json"])
-    if r.returncode != 0 or not r.stdout.strip():
-        return None
-    try:
-        data = json.loads(r.stdout)
-        row = data[0] if isinstance(data, list) and data else data if isinstance(data, dict) else None
-        if not isinstance(row, dict):
-            return None
-        st = row.get("status")
-        if isinstance(st, dict):
-            stage = st.get("stage")
-            if isinstance(stage, str):
-                return stage.lower()
-        if isinstance(st, str):
-            return st.lower()
-    except json.JSONDecodeError:
-        pass
-    return None
-
-
-def _wait_job(job_id: str, poll_s: float = 45.0, label: str = "") -> int:
-    print(f"[push_to_jobs] 轮询 Job: {job_id} ({label})")
-    terminal_ok = ("completed", "succeeded", "success", "done")
-    terminal_bad = ("failed", "error", "cancelled", "canceled", "stopped")
-    while True:
-        st = _job_status(job_id)
-        if st:
-            print(f"[push_to_jobs] 状态: {st}")
-            if st in terminal_ok:
-                return 0
-            if st in terminal_bad:
-                return 1
-        time.sleep(poll_s)
-
-
-def _submit_detach(cmd: list[str]) -> tuple[int, str]:
-    r = _run_capture(cmd)
-    out = (r.stdout or "") + "\n" + (r.stderr or "")
-    if r.returncode != 0:
-        print(out)
-        return r.returncode, ""
-    jid = _parse_detach_job_id(r.stdout or "", r.stderr or "")
-    if jid:
-        print(f"[push_to_jobs] Job ID: {jid}")
-    else:
-        print(out)
-    return r.returncode, jid or ""
-
-
-def main() -> None:
-    p = argparse.ArgumentParser(description="Submit HF Jobs: copy / extract / train.")
-    p.add_argument("--bucket", default=DEFAULT_BUCKET)
-    p.add_argument("--code-repo", default=DEFAULT_CODE, dest="code_repo")
-    p.add_argument("--weights-repo", default=DEFAULT_WEIGHTS, dest="weights_repo")
-    p.add_argument("--src", default=DEFAULT_SRC, dest="src")
-    p.add_argument("--mirror-subpath", default=DEFAULT_MIRROR, dest="mirror_subpath")
-    p.add_argument(
-        "--extracted-subpath",
-        default=DEFAULT_EXTRACTED,
-        dest="extracted_subpath",
-        help="解压/训练用的数据集根路径（默认 extracted/cosmos_hub，与 mirror 分离）",
-    )
-    p.add_argument(
-        "--cache-subpath",
-        default=DEFAULT_CACHE,
-        dest="cache_subpath",
-        help="Bucket 内缓存根目录（默认 cache，可整体删除）",
-    )
-    p.add_argument("--timeout", default=DEFAULT_TIMEOUT)
-    p.add_argument("--detach", action="store_true")
-    p.add_argument("--train-image", default=TRAIN_IMAGE, dest="train_image")
-    p.add_argument("--train-flavor", default=TRAIN_FLAVOR, dest="train_flavor")
-    p.add_argument(
-        "action",
-        choices=("copy", "extract", "train", "print-plan", "copy-extract"),
-    )
-    args = p.parse_args()
-
-    builders = {
-        "copy": build_copy_cmd,
-        "extract": build_extract_cmd,
-        "train": build_train_cmd,
-    }
-    if args.action == "print-plan":
-        for name, b in builders.items():
-            print(f"--- {name} ---")
-            print(" ".join(shlex.quote(c) for c in b(args)))
-            print()
-        return
-
-    if args.action == "copy-extract":
-        cargs = argparse.Namespace(**vars(args))
-        cargs.detach = True
-        rc, jid = _submit_detach(build_copy_cmd(cargs))
-        if rc != 0:
-            sys.exit(rc)
-        if not jid:
-            print("[push_to_jobs] 未解析到 copy 的 Job ID。请到 Hub Jobs 查看后手动: python scripts/push_to_jobs.py extract")
-            sys.exit(1)
-        rc = _wait_job(jid, label="copy to bucket")
-        if rc != 0:
-            sys.exit(rc)
-        sys.exit(_run(build_extract_cmd(args)))
-
-    sys.exit(_run(builders[args.action](args)))
-
-
-if __name__ == "__main__":
-    main()
+"""提交 Hugging Face Jobs：数据集服务端 copy → CPU 挂载解压 → A10G-Large 正式训练。
+
+**持久化只在 Bucket 上**（挂载如 ``/mnt/wjad``）。容器本地只应保留镜像层；
+``pip`` / ``TMPDIR`` / ``HF_HOME`` 等均指向 ``cache/`` ，避免 ephemeral 配额（如 50G）被撑爆。
+
+Bucket 目录约定（相对挂载根）::
+
+- ``mirror/cosmos_hub/`` — ``copy_files`` 得到的 Hub 数据集镜像；**只作解压源**，不把解压产物写回这里。
+- ``extracted/cosmos_hub/`` — **仅解压输出**，相对路径与源一致，训练 ``--data_root`` 指向这里。
+- ``cache/`` — pip、TMPDIR、HF/torch/transformers 缓存及 ``wjad_repo``（clone 代码），**可整目录删** 以清缓存或换代码版本。
+- ``runs/current/`` — checkpoint（仍在 Bucket）；另可 ``upload_file`` 到 Hub model repo。
+
+常用命令::
+
+    python scripts/push_to_jobs.py copy-extract   # 先 copy（等待成功）再 submit extract（前台跑完日志）
+
+- ``--replace-extracted`` — 解压前 **删除** ``extracted/...`` 下全部内容（含 ``_wjad_extract_state``），再全量重解压。
+
+单独步骤可加 ``--detach``。Windows 上会使用 ``sys.executable`` 同目录的 ``hf.exe``。
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+DEFAULT_BUCKET = "fuzirui/WJAD"
+DEFAULT_CODE = "fuzirui/WJAD"
+DEFAULT_WEIGHTS = "fuzirui/WJAD"
+DEFAULT_SRC = "hf://datasets/nvidia/PhysicalAI-Autonomous-Vehicle-Cosmos-Drive-Dreams/"
+DEFAULT_MIRROR = "mirror/cosmos_hub"
+DEFAULT_EXTRACTED = "extracted/cosmos_hub"
+DEFAULT_CACHE = "cache"
+DEFAULT_TIMEOUT = "7d"
+TRAIN_IMAGE = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime"
+TRAIN_FLAVOR = "a10g-large"
+
+
+def _hf_executable() -> str:
+    sibling = Path(sys.executable).parent / ("hf.exe" if sys.platform == "win32" else "hf")
+    if sibling.is_file():
+        return str(sibling.resolve())
+    w = shutil.which("hf")
+    return w if w else "hf"
+
+
+def _hf_jobs_cmd(parts: list[str]) -> list[str]:
+    if parts and parts[0] == "hf":
+        return [_hf_executable(), *parts[1:]]
+    return parts
+
+
+def _insert_detach(cmd: list[str], detach: bool) -> None:
+    if not detach:
+        return
+    idx = cmd.index("run") + 1
+    cmd.insert(idx, "--detach")
+
+
+def build_copy_cmd(args: argparse.Namespace) -> list[str]:
+    dst = f"hf://buckets/{args.bucket}/{args.mirror_subpath.rstrip('/')}/"
+    one_py = (
+        "import subprocess,sys;"
+        "subprocess.check_call([sys.executable,'-m','pip','install','-q','-U','huggingface_hub']);"
+        "from huggingface_hub import copy_files;"
+        f"copy_files({args.src!r},{dst!r})"
+    )
+    cmd = [
+        "hf",
+        "jobs",
+        "run",
+        "--flavor",
+        "cpu-basic",
+        "--timeout",
+        args.timeout,
+        "--secrets",
+        "HF_TOKEN",
+        "python:3.12",
+        "python",
+        "-c",
+        one_py,
+    ]
+    _insert_detach(cmd, args.detach)
+    return cmd
+
+
+def _bucket_cache_env_sh(cache_subpath: str) -> str:
+    """单行：pip / 临时目录 / HF 相关缓存全部落在挂载盘 cache/ 下。"""
+    c = f"/mnt/wjad/{cache_subpath}"
+    return (
+        f"export WJAD_CACHE_ROOT={c} && "
+        f"mkdir -p {c}/pip {c}/tmp {c}/hf {c}/transformers {c}/torch {c}/datasets {c}/xdg && "
+        f"export PIP_CACHE_DIR={c}/pip && export TMPDIR={c}/tmp && export TEMP={c}/tmp && export TMP={c}/tmp && "
+        f"export HF_HOME={c}/hf && export TRANSFORMERS_CACHE={c}/transformers && export TORCH_HOME={c}/torch && "
+        f"export HF_DATASETS_CACHE={c}/datasets && export XDG_CACHE_HOME={c}/xdg"
+    )
+
+
+def build_extract_cmd(args: argparse.Namespace) -> list[str]:
+    vol = f"hf://buckets/{args.bucket}:/mnt/wjad"
+    cache_setup = _bucket_cache_env_sh(args.cache_subpath)
+    clone_dir = f"/mnt/wjad/{args.cache_subpath}/wjad_repo"
+    ext_dir = f"/mnt/wjad/{args.extracted_subpath}"
+    steps: list[str] = [
+        "set -e",
+        "export WJAD_BUCKET_MOUNT=/mnt/wjad",
+        cache_setup,
+    ]
+    if getattr(args, "replace_extracted", False):
+        steps.append(
+            f'echo "[extract] --replace-extracted: wipe {ext_dir}" && rm -rf {ext_dir} && mkdir -p {ext_dir}'
+        )
+    steps.extend(
+        [
+            "command -v git >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends git)",
+            "pip install -q -U huggingface_hub",
+            f"rm -rf {clone_dir} && git clone https://oauth2:$HF_TOKEN@huggingface.co/{args.code_repo} {clone_dir}",
+            f"cd {clone_dir}",
+            "python scripts/jobs_extract_archives.py "
+            f"--scan-root /mnt/wjad/{args.mirror_subpath} --out-root {ext_dir}",
+        ]
+    )
+    inner = " && ".join(steps)
+    cmd = [
+        "hf",
+        "jobs",
+        "run",
+        "-v",
+        vol,
+        "--flavor",
+        "cpu-basic",
+        "--timeout",
+        args.timeout,
+        "--secrets",
+        "HF_TOKEN",
+        "python:3.12",
+        "sh",
+        "-c",
+        inner,
+    ]
+    _insert_detach(cmd, args.detach)
+    return cmd
+
+
+def build_train_cmd(args: argparse.Namespace) -> list[str]:
+    vol = f"hf://buckets/{args.bucket}:/mnt/wjad"
+    cache_setup = _bucket_cache_env_sh(args.cache_subpath)
+    clone_dir = f"/mnt/wjad/{args.cache_subpath}/wjad_repo"
+    inner = " && ".join(
+        [
+            "set -e",
+            "export WJAD_BUCKET_MOUNT=/mnt/wjad",
+            cache_setup,
+            f"export WJAD_HUB_REPO={args.weights_repo}",
+            f"export WJAD_DATA_ROOT=/mnt/wjad/{args.extracted_subpath}",
+            "export WJAD_OUTPUT_DIR=/mnt/wjad/runs/current",
+            "command -v git >/dev/null 2>&1 || (apt-get update && apt-get install -y --no-install-recommends git)",
+            "pip install -q -U huggingface_hub",
+            f"rm -rf {clone_dir} && git clone https://oauth2:$HF_TOKEN@huggingface.co/{args.code_repo} {clone_dir}",
+            f"cd {clone_dir}",
+            "pip install -q -U pip",
+            "pip install -q -e .",
+            "bash scripts/jobs_entry_train.sh",
+        ]
+    )
+    cmd = [
+        "hf",
+        "jobs",
+        "run",
+        "-v",
+        vol,
+        "--flavor",
+        args.train_flavor,
+        "--timeout",
+        args.timeout,
+        "--secrets",
+        "HF_TOKEN",
+        args.train_image,
+        "bash",
+        "-lc",
+        inner,
+    ]
+    _insert_detach(cmd, args.detach)
+    return cmd
+
+
+def _run(cmd: list[str]) -> int:
+    print("[push_to_jobs] $", " ".join(shlex.quote(c) for c in cmd))
+    cmd = _hf_jobs_cmd(cmd)
+    return subprocess.call(cmd)
+
+
+def _run_capture(cmd: list[str]) -> subprocess.CompletedProcess:
+    cmd = _hf_jobs_cmd(cmd)
+    print("[push_to_jobs] $", " ".join(shlex.quote(c) for c in cmd))
+    return subprocess.run(cmd, capture_output=True, text=True)
+
+
+def _parse_detach_job_id(stdout: str, stderr: str) -> str | None:
+    text = (stdout + "\n" + stderr).strip()
+    m = re.search(r"\bID:\s*([a-fA-F0-9]{12,})\b", text)
+    if m:
+        return m.group(1)
+    m = re.search(r"/jobs/[^/\s]+/([a-fA-F0-9]{12,})", text)
+    if m:
+        return m.group(1)
+    for line in text.splitlines():
+        line = line.strip()
+        low = line.lower()
+        if "job" in low and "id" in low:
+            parts = line.replace(":", " ").split()
+            for i, p in enumerate(parts):
+                if p.lower() == "id" and i + 1 < len(parts):
+                    return parts[i + 1].strip().rstrip(",")
+    return None
+
+
+def _job_status(job_id: str) -> str | None:
+    r = _run_capture(["hf", "jobs", "inspect", job_id, "--json"])
+    if r.returncode != 0 or not r.stdout.strip():
+        return None
+    try:
+        data = json.loads(r.stdout)
+        row = data[0] if isinstance(data, list) and data else data if isinstance(data, dict) else None
+        if not isinstance(row, dict):
+            return None
+        st = row.get("status")
+        if isinstance(st, dict):
+            stage = st.get("stage")
+            if isinstance(stage, str):
+                return stage.lower()
+        if isinstance(st, str):
+            return st.lower()
+    except json.JSONDecodeError:
+        pass
+    return None
+
+
+def _wait_job(job_id: str, poll_s: float = 45.0, label: str = "") -> int:
+    print(f"[push_to_jobs] 轮询 Job: {job_id} ({label})")
+    terminal_ok = ("completed", "succeeded", "success", "done")
+    terminal_bad = ("failed", "error", "cancelled", "canceled", "stopped")
+    while True:
+        st = _job_status(job_id)
+        if st:
+            print(f"[push_to_jobs] 状态: {st}")
+            if st in terminal_ok:
+                return 0
+            if st in terminal_bad:
+                return 1
+        time.sleep(poll_s)
+
+
+def _submit_detach(cmd: list[str]) -> tuple[int, str]:
+    r = _run_capture(cmd)
+    out = (r.stdout or "") + "\n" + (r.stderr or "")
+    if r.returncode != 0:
+        print(out)
+        return r.returncode, ""
+    jid = _parse_detach_job_id(r.stdout or "", r.stderr or "")
+    if jid:
+        print(f"[push_to_jobs] Job ID: {jid}")
+    else:
+        print(out)
+    return r.returncode, jid or ""
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(description="Submit HF Jobs: copy / extract / train.")
+    p.add_argument("--bucket", default=DEFAULT_BUCKET)
+    p.add_argument("--code-repo", default=DEFAULT_CODE, dest="code_repo")
+    p.add_argument("--weights-repo", default=DEFAULT_WEIGHTS, dest="weights_repo")
+    p.add_argument("--src", default=DEFAULT_SRC, dest="src")
+    p.add_argument("--mirror-subpath", default=DEFAULT_MIRROR, dest="mirror_subpath")
+    p.add_argument(
+        "--extracted-subpath",
+        default=DEFAULT_EXTRACTED,
+        dest="extracted_subpath",
+        help="解压/训练用的数据集根路径（默认 extracted/cosmos_hub，与 mirror 分离）",
+    )
+    p.add_argument(
+        "--cache-subpath",
+        default=DEFAULT_CACHE,
+        dest="cache_subpath",
+        help="Bucket 内缓存根目录（默认 cache，可整体删除）",
+    )
+    p.add_argument("--timeout", default=DEFAULT_TIMEOUT)
+    p.add_argument("--detach", action="store_true")
+    p.add_argument(
+        "--replace-extracted",
+        action="store_true",
+        help="仅 extract：先 rm -rf extracted 目标目录再解压，完全替换旧解压结果",
+    )
+    p.add_argument("--train-image", default=TRAIN_IMAGE, dest="train_image")
+    p.add_argument("--train-flavor", default=TRAIN_FLAVOR, dest="train_flavor")
+    p.add_argument(
+        "action",
+        choices=("copy", "extract", "train", "print-plan", "copy-extract"),
+    )
+    args = p.parse_args()
+
+    builders = {
+        "copy": build_copy_cmd,
+        "extract": build_extract_cmd,
+        "train": build_train_cmd,
+    }
+    if args.action == "print-plan":
+        for name, b in builders.items():
+            print(f"--- {name} ---")
+            print(" ".join(shlex.quote(c) for c in b(args)))
+            print()
+        return
+
+    if args.action == "copy-extract":
+        cargs = argparse.Namespace(**vars(args))
+        cargs.detach = True
+        rc, jid = _submit_detach(build_copy_cmd(cargs))
+        if rc != 0:
+            sys.exit(rc)
+        if not jid:
+            print("[push_to_jobs] 未解析到 copy 的 Job ID。请到 Hub Jobs 查看后手动: python scripts/push_to_jobs.py extract")
+            sys.exit(1)
+        rc = _wait_job(jid, label="copy to bucket")
+        if rc != 0:
+            sys.exit(rc)
+        sys.exit(_run(build_extract_cmd(args)))
+
+    sys.exit(_run(builders[args.action](args)))
+
+
+if __name__ == "__main__":
+    main()