Spaces:
Sleeping
Sleeping
File size: 2,415 Bytes
0d542b0 4424ac7 0d542b0 4424ac7 0d542b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | #!/usr/bin/env python3
"""
Refresh the bundled BESS list snapshot.
Downloads the AEMO Generation Information XLSX, parses it into the same
shape served by /api/bess, and writes app/data/bess_list.json. This file is
the runtime fallback used by gen_info_fetcher.py when the live AEMO fetch
is blocked (e.g. WAF 403 from the HuggingFace egress IP).
A small bess_list_meta.json sibling records when the snapshot was fetched
so the frontend banner can display its age.
The raw XLSX itself is intentionally NOT committed: HuggingFace Spaces
rejects binary files in non-Xet storage, and the parsed JSON snapshot is a
strictly equivalent fallback (same parser, same source).
Run from the repo root:
python scripts/refresh_bess_list.py
Intended to be invoked by a scheduled GitHub Action (see
.github/workflows/refresh_bess_list.yml).
"""
from __future__ import annotations
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
import httpx
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
from app.services.gen_info_fetcher import XLSX_URL, _BROWSER_HEADERS, _parse_xlsx
DATA_DIR = REPO_ROOT / "app" / "data"
JSON_OUT = DATA_DIR / "bess_list.json"
META_OUT = DATA_DIR / "bess_list_meta.json"
def main() -> int:
print(f"Downloading {XLSX_URL}")
with httpx.Client(timeout=60, follow_redirects=True) as client:
resp = client.get(XLSX_URL, headers=_BROWSER_HEADERS)
resp.raise_for_status()
xlsx_bytes = resp.content
print(f"Got {len(xlsx_bytes):,} bytes")
parsed = _parse_xlsx(xlsx_bytes)
total = sum(len(v) for v in parsed.values())
print(f"Parsed {total} BESS units across {sum(1 for v in parsed.values() if v)} states")
if total == 0:
print("ERROR: parsed XLSX yielded zero BESS units — refusing to overwrite snapshot")
return 1
DATA_DIR.mkdir(parents=True, exist_ok=True)
JSON_OUT.write_text(json.dumps(parsed, indent=2, sort_keys=True) + "\n")
META_OUT.write_text(json.dumps({
"fetched_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"source_url": XLSX_URL,
"xlsx_bytes": len(xlsx_bytes),
"unit_count": total,
}, indent=2) + "\n")
print(f"Wrote {JSON_OUT.relative_to(REPO_ROOT)}")
print(f"Wrote {META_OUT.relative_to(REPO_ROOT)}")
return 0
if __name__ == "__main__":
sys.exit(main())
|