""" Orchestrator: run all data processing steps in sequence and report status. Usage: python -m src.data_processing.process_all """ import logging import time import sys from pathlib import Path logger = logging.getLogger(__name__) def process_all(): """Run all data processing steps in order.""" start = time.time() print("=" * 60) print(" InstaWarn — Data Processing Pipeline") print("=" * 60) print() results = {} # ── Step 1: Administrative Boundaries ───────────────────────────────── print("[1/5] Processing administrative boundaries...") try: from src.data_processing.load_boundaries import load_and_filter_boundaries unions = load_and_filter_boundaries() results["boundaries"] = f"✅ {len(unions)} unions loaded" except Exception as e: results["boundaries"] = f"❌ Failed: {e}" print(f" ERROR: {e}") print(" This is a BLOCKER — cannot continue without boundaries.") _print_summary(results, start) return False print() # ── Step 2: Cyclone Track ───────────────────────────────────────────── print("[2/5] Processing cyclone track data...") try: from src.data_processing.load_cyclone_track import load_cyclone_track track = load_cyclone_track() results["cyclone_track"] = f"✅ {len(track)} track points" except Exception as e: results["cyclone_track"] = f"❌ Failed: {e}" print(f" ERROR: {e}") print() # ── Step 3: OSM Infrastructure ──────────────────────────────────────── print("[3/5] Extracting OSM infrastructure data...") try: from src.data_processing.extract_osm_data import extract_all_osm_data osm = extract_all_osm_data() counts = {k: len(v) for k, v in osm.items()} results["osm_data"] = f"✅ {counts}" except Exception as e: results["osm_data"] = f"⚠️ Partial/Failed: {e}" print(f" WARNING: {e}") print() # ── Step 4: Population ──────────────────────────────────────────────── print("[4/5] Processing population data...") try: from src.data_processing.load_population import load_population pop = load_population() total = pop["population"].sum() results["population"] = f"✅ {len(pop)} unions, total pop: {total:,}" except Exception as e: results["population"] = f"⚠️ Failed: {e}" print(f" WARNING: {e}") print() # ── Step 5: Shelters ────────────────────────────────────────────────── print("[5/5] Processing shelter data...") try: from src.data_processing.load_shelters import load_shelters shelters = load_shelters() total_cap = shelters["capacity"].sum() results["shelters"] = f"✅ {len(shelters)} shelters, capacity: {total_cap:,}" except Exception as e: results["shelters"] = f"⚠️ Failed: {e}" print(f" WARNING: {e}") print() _print_summary(results, start) return True def _print_summary(results, start_time): """Print processing summary.""" elapsed = time.time() - start_time print("=" * 60) print(" Processing Summary") print("=" * 60) for step, status in results.items(): print(f" {step}: {status}") print(f"\n Total time: {elapsed:.1f}s") print("=" * 60) # List processed files processed_dir = Path(__file__).parent.parent.parent / "data" / "processed" if processed_dir.exists(): files = sorted(processed_dir.glob("*")) print(f"\n Processed files ({len(files)}):") for f in files: size_kb = f.stat().st_size / 1024 print(f" {f.name} ({size_kb:.1f} KB)") print() if __name__ == "__main__": logging.basicConfig( level=logging.INFO, format="%(levelname)s: %(message)s", ) success = process_all() sys.exit(0 if success else 1)