File size: 3,123 Bytes
6371d28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """
Push to both Hugging Face Dataset and Space.
Dataset: https://huggingface.co/datasets/scholo/MMB_dataset
Space: https://huggingface.co/spaces/scholo/Datasetviewer
Usage:
pip install datasets pillow huggingface_hub
huggingface-cli login
python scripts/deploy_both.py
python scripts/deploy_both.py --dataset-only # Only push dataset
python scripts/deploy_both.py --space-only # Only push Space
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
HF_DATASET_DIR = PROJECT_ROOT / "hf_dataset"
DATASET_REPO = "scholo/MMB_dataset"
SPACE_REPO = "scholo/Datasetviewer"
def find_hf_dataset_csv() -> Path | None:
"""Find CSV in hf_dataset/ (root or in subdirs)."""
if not HF_DATASET_DIR.exists():
return None
csvs = list(HF_DATASET_DIR.rglob("*.csv"))
with_q = [p for p in csvs if "question" in p.name.lower()]
return (with_q[0] if with_q else csvs[0]) if csvs else None
def run(cmd: list[str], cwd: Path | None = None) -> bool:
r = subprocess.run(cmd, cwd=cwd or PROJECT_ROOT)
return r.returncode == 0
def push_dataset() -> bool:
print("=== Pushing dataset to", DATASET_REPO, "===")
csv_path = find_hf_dataset_csv()
if not csv_path:
print(f"Error: No CSV in {HF_DATASET_DIR}. Add image_mapping_with_questions.csv + images/ + scenes/", file=sys.stderr)
return False
print(f"Using {csv_path.relative_to(PROJECT_ROOT)}")
return run([
sys.executable,
str(PROJECT_ROOT / "scripts" / "upload_to_huggingface.py"),
str(csv_path),
"--repo-id", DATASET_REPO,
])
def push_space() -> bool:
print("=== Pushing Space to", SPACE_REPO, "===")
try:
from huggingface_hub import HfApi
except ImportError:
print("Install huggingface_hub: pip install huggingface_hub", file=sys.stderr)
return False
api = HfApi()
try:
api.upload_folder(
folder_path=str(PROJECT_ROOT),
path_in_repo="",
repo_id=SPACE_REPO,
repo_type="space",
ignore_patterns=[
"__pycache__", "*.pyc", ".git",
"scripts", "requirements-upload.txt",
"hf_dataset", # Dataset has its own folder; Space uses data/
"*.zip", ".cursor", "*.pyo",
],
commit_message="Deploy MMB Dataset Visualizer",
)
print("Space pushed. View at: https://huggingface.co/spaces/" + SPACE_REPO)
return True
except Exception as e:
print(f"Error pushing Space: {e}", file=sys.stderr)
return False
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-only", action="store_true")
parser.add_argument("--space-only", action="store_true")
args = parser.parse_args()
ok = True
if not args.space_only:
ok = push_dataset() and ok
if not args.dataset_only:
ok = push_space() and ok
sys.exit(0 if ok else 1)
if __name__ == "__main__":
main()
|