"""Fetch InstaWarn datasets from Hugging Face when local data is absent.""" from __future__ import annotations import os from src.config import DATA_DIR, PROCESSED_UNIONS, PROJECT_ROOT DATASET_REPO = os.environ.get( "INSTAWARN_DATASET_REPO", "jubayerahmad/instawarn-data", ) MARKER = PROCESSED_UNIONS def data_is_present() -> bool: return MARKER.is_file() def ensure_data(*, force: bool = False) -> bool: """Download dataset files if missing. Returns True when data is available.""" if os.environ.get("INSTAWARN_SKIP_DATA_DOWNLOAD", "").lower() in ("1", "true", "yes"): return data_is_present() if not force and data_is_present(): return True from huggingface_hub import snapshot_download snapshot_download( repo_id=DATASET_REPO, repo_type="dataset", local_dir=str(PROJECT_ROOT), allow_patterns=["data/**"], ) return data_is_present()