scripts/refresh_bess_list.py

#!/usr/bin/env python3
"""
Refresh the bundled BESS list snapshot.

Downloads the AEMO Generation Information XLSX, parses it into the same
shape served by /api/bess, and writes app/data/bess_list.json. This file is
the runtime fallback used by gen_info_fetcher.py when the live AEMO fetch
is blocked (e.g. WAF 403 from the HuggingFace egress IP).

A small bess_list_meta.json sibling records when the snapshot was fetched
so the frontend banner can display its age.

The raw XLSX itself is intentionally NOT committed: HuggingFace Spaces
rejects binary files in non-Xet storage, and the parsed JSON snapshot is a
strictly equivalent fallback (same parser, same source).

Run from the repo root:
    python scripts/refresh_bess_list.py

Intended to be invoked by a scheduled GitHub Action (see
.github/workflows/refresh_bess_list.yml).
"""
from __future__ import annotations

import json
import sys
from datetime import datetime, timezone
from pathlib import Path

import httpx

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))

from app.services.gen_info_fetcher import XLSX_URL, _BROWSER_HEADERS, _parse_xlsx

DATA_DIR = REPO_ROOT / "app" / "data"
JSON_OUT = DATA_DIR / "bess_list.json"
META_OUT = DATA_DIR / "bess_list_meta.json"


def main() -> int:
    print(f"Downloading {XLSX_URL}")
    with httpx.Client(timeout=60, follow_redirects=True) as client:
        resp = client.get(XLSX_URL, headers=_BROWSER_HEADERS)
        resp.raise_for_status()
    xlsx_bytes = resp.content
    print(f"Got {len(xlsx_bytes):,} bytes")

    parsed = _parse_xlsx(xlsx_bytes)
    total = sum(len(v) for v in parsed.values())
    print(f"Parsed {total} BESS units across {sum(1 for v in parsed.values() if v)} states")
    if total == 0:
        print("ERROR: parsed XLSX yielded zero BESS units — refusing to overwrite snapshot")
        return 1

    DATA_DIR.mkdir(parents=True, exist_ok=True)
    JSON_OUT.write_text(json.dumps(parsed, indent=2, sort_keys=True) + "\n")
    META_OUT.write_text(json.dumps({
        "fetched_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "source_url": XLSX_URL,
        "xlsx_bytes": len(xlsx_bytes),
        "unit_count": total,
    }, indent=2) + "\n")

    print(f"Wrote {JSON_OUT.relative_to(REPO_ROOT)}")
    print(f"Wrote {META_OUT.relative_to(REPO_ROOT)}")
    return 0


if __name__ == "__main__":
    sys.exit(main())