Drop XLSX mirror layer to satisfy HuggingFace binary-file policy

The previous commit was rejected by HuggingFace ("push was rejected because
it contains binary files. Please use Xet to store binary files. Offending
files: app/data/aemo_gen_info.xlsx") so the sync_to_hf workflow failed
and the Space stayed on the old code.

The mirror layer (live -> raw.githubusercontent.com XLSX -> snapshot) was
redundant: the daily CI commits the parsed JSON snapshot in the same step
as the XLSX, so they always carried identical freshness. Collapsing to a
two-tier fallback (live -> snapshot) keeps the user-visible behaviour
identical while removing the binary file that HF rejects.

- gen_info_fetcher.py: remove XLSX_MIRROR_URL and the mirror branch in
fetch_bess_list. BessListResult.source is now "live" | "snapshot".
- scripts/refresh_bess_list.py: stop writing app/data/aemo_gen_info.xlsx.
Only bess_list.json and bess_list_meta.json are produced.
- .github/workflows/refresh_bess_list.yml: only watch bess_list.json for
changes.
- app/data/aemo_gen_info.xlsx: removed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/workflows/refresh_bess_list.yml

@@ -31,14 +31,15 @@ jobs:
       - name: Commit if changed
         run: |
           set -e
-          # Only commit if the parsed JSON or raw XLSX changed.
-          # The meta file changes every run (timestamp); ignore it on its own.
-          if git diff --quiet -- app/data/bess_list.json app/data/aemo_gen_info.xlsx; then
-            echo "No change in bess_list.json or aemo_gen_info.xlsx; skipping commit."
+          # Only commit if the parsed JSON changed. The meta file changes every
+          # run (timestamp); committing it alone would churn master without
+          # delivering new data.
+          if git diff --quiet -- app/data/bess_list.json; then
+            echo "No change in bess_list.json; skipping commit."
             exit 0
           fi
           git config user.name  "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add app/data/bess_list.json app/data/aemo_gen_info.xlsx app/data/bess_list_meta.json
+          git add app/data/bess_list.json app/data/bess_list_meta.json
           git commit -m "Refresh BESS list snapshot from AEMO XLSX"
           git push origin master

app/services/gen_info_fetcher.py

@@ -5,10 +5,10 @@ Sources tried in order on every cache miss:
 
   1. live      — direct download from www.aemo.com.au (subject to WAF blocks
                  from datacentre IPs; see _BROWSER_HEADERS).
-  2. mirror    — raw.githubusercontent.com copy committed daily by the
-                 refresh_bess_list workflow.
-  3. snapshot  — the parsed bess_list.json bundled in the repo (also
-                 written by refresh_bess_list).
+  2. snapshot  — the parsed bess_list.json bundled in the repo, refreshed
+                 daily by .github/workflows/refresh_bess_list.yml running
+                 scripts/refresh_bess_list.py from GitHub-hosted runners
+                 (whose egress is not blocked by AEMO's WAF).
 
 The first successful step is cached in memory for 24 hours.  The result
 carries a `source` field and any user-facing `warnings` so the API can
@@ -32,14 +32,6 @@ XLSX_URL = (
     "?rev=1f6bccf827284f9fb6d6f3ae56ed3fe9&sc_lang=en"
 )
 
-# raw.githubusercontent.com copy of the XLSX, refreshed daily by
-# .github/workflows/refresh_bess_list.yml. Used as the second-choice source
-# when the live AEMO fetch fails (e.g. WAF 403 from the HuggingFace egress IP).
-XLSX_MIRROR_URL = (
-    "https://raw.githubusercontent.com/pourmousavi/BESS-SCADA-Data"
-    "/master/app/data/aemo_gen_info.xlsx"
-)
-
 # AEMO's Azure Front Door WAF on www.aemo.com.au returns 403 to requests that
 # look like default Python clients. Sending a full browser-style header set
 # bypasses the simpler bot-detection rules. (Doesn't help against pure
@@ -94,8 +86,8 @@ CACHE_TTL = timedelta(hours=24)
 class BessListResult:
     """Parsed BESS list plus provenance for the API response."""
     states: dict[str, list[dict]]
-    source: str            # "live" | "mirror" | "snapshot"
-    fetched_at: datetime   # for the live/mirror path: now; for snapshot: snapshot timestamp
+    source: str            # "live" | "snapshot"
+    fetched_at: datetime   # for the live path: now; for snapshot: snapshot timestamp
     warnings: list[str] = field(default_factory=list)
 
 
@@ -251,10 +243,14 @@ def _load_snapshot() -> BessListResult:
 
 async def fetch_bess_list() -> BessListResult:
     """
-    Return the BESS list, trying live AEMO → GitHub mirror → bundled snapshot.
+    Return the BESS list, trying live AEMO → bundled snapshot.
+
+    The snapshot is refreshed daily by the refresh_bess_list workflow from
+    GitHub-hosted runners (whose egress is not blocked by AEMO), so even
+    when the live AEMO fetch fails the fallback is at most ~24 hours old.
 
-    Uses a 24-hour in-memory cache (per source). The result carries provenance
-    so the API can surface a warning when serving non-live data.
+    Uses a 24-hour in-memory cache. The result carries provenance so the
+    API can surface a warning when serving non-live data.
     """
     global _cache
 
@@ -268,43 +264,25 @@ async def fetch_bess_list() -> BessListResult:
                          now - cached_at, cached.source)
             return cached
 
-    async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
-        # 1. Live AEMO
-        try:
-            logger.info("Downloading AEMO XLSX (live): %s", XLSX_URL)
+    # 1. Live AEMO
+    try:
+        logger.info("Downloading AEMO XLSX (live): %s", XLSX_URL)
+        async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
             states = await _download_and_parse(XLSX_URL, client)
-            result = BessListResult(states=states, source="live", fetched_at=now)
-            _cache = (result, now)
-            return result
-        except Exception as exc:
-            logger.warning("Live AEMO XLSX fetch failed: %s", exc)
+        result = BessListResult(states=states, source="live", fetched_at=now)
+        _cache = (result, now)
+        return result
+    except Exception as exc:
+        logger.warning("Live AEMO XLSX fetch failed: %s", exc)
 
-        # 2. GitHub raw mirror (refreshed daily by CI)
-        try:
-            logger.info("Downloading AEMO XLSX (mirror): %s", XLSX_MIRROR_URL)
-            states = await _download_and_parse(XLSX_MIRROR_URL, client)
-            result = BessListResult(
-                states=states,
-                source="mirror",
-                fetched_at=now,
-                warnings=[
-                    "Live AEMO BESS list is unavailable; showing a copy mirrored "
-                    "from GitHub (updated daily)."
-                ],
-            )
-            _cache = (result, now)
-            return result
-        except Exception as exc:
-            logger.warning("Mirror XLSX fetch failed: %s", exc)
-
-    # 3. Bundled snapshot — last resort. Always serve something rather than 5xx.
+    # 2. Bundled snapshot — refreshed daily by CI from GH-hosted runners.
     try:
         snapshot = _load_snapshot()
         age = now - snapshot.fetched_at
         days = max(age.days, 0)
         snapshot.warnings = [
-            f"Live AEMO BESS list and mirror are both unavailable; "
-            f"showing bundled snapshot from {snapshot.fetched_at:%Y-%m-%d} "
+            f"Live AEMO BESS list is unavailable; showing the daily snapshot "
+            f"committed on {snapshot.fetched_at:%Y-%m-%d} "
             f"({days} day{'s' if days != 1 else ''} old)."
         ]
         _cache = (snapshot, now)

scripts/refresh_bess_list.py

@@ -3,14 +3,16 @@
 Refresh the bundled BESS list snapshot.
 
 Downloads the AEMO Generation Information XLSX, parses it into the same
-shape served by /api/bess, and writes two files into app/data/:
+shape served by /api/bess, and writes app/data/bess_list.json. This file is
+the runtime fallback used by gen_info_fetcher.py when the live AEMO fetch
+is blocked (e.g. WAF 403 from the HuggingFace egress IP).
 
-  bess_list.json      — parsed snapshot used as runtime fallback when the
-                        live AEMO fetch is blocked (e.g. WAF 403 from the
-                        HuggingFace egress IP).
-  aemo_gen_info.xlsx  — the raw XLSX bytes, so gen_info_fetcher.py can also
-                        fall back to fetching it from raw.githubusercontent.com
-                        and parse a fresher copy at runtime.
+A small bess_list_meta.json sibling records when the snapshot was fetched
+so the frontend banner can display its age.
+
+The raw XLSX itself is intentionally NOT committed: HuggingFace Spaces
+rejects binary files in non-Xet storage, and the parsed JSON snapshot is a
+strictly equivalent fallback (same parser, same source).
 
 Run from the repo root:
     python scripts/refresh_bess_list.py
@@ -34,7 +36,6 @@ from app.services.gen_info_fetcher import XLSX_URL, _BROWSER_HEADERS, _parse_xls
 
 DATA_DIR = REPO_ROOT / "app" / "data"
 JSON_OUT = DATA_DIR / "bess_list.json"
-XLSX_OUT = DATA_DIR / "aemo_gen_info.xlsx"
 META_OUT = DATA_DIR / "bess_list_meta.json"
 
 
@@ -54,7 +55,6 @@ def main() -> int:
         return 1
 
     DATA_DIR.mkdir(parents=True, exist_ok=True)
-    XLSX_OUT.write_bytes(xlsx_bytes)
     JSON_OUT.write_text(json.dumps(parsed, indent=2, sort_keys=True) + "\n")
     META_OUT.write_text(json.dumps({
         "fetched_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
@@ -64,7 +64,6 @@ def main() -> int:
     }, indent=2) + "\n")
 
     print(f"Wrote {JSON_OUT.relative_to(REPO_ROOT)}")
-    print(f"Wrote {XLSX_OUT.relative_to(REPO_ROOT)}")
     print(f"Wrote {META_OUT.relative_to(REPO_ROOT)}")
     return 0