Spaces:
Running
Running
File size: 3,950 Bytes
1e10174 18a63f1 1e10174 18a63f1 1e10174 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """Regenerate the README results table from the trained model leaderboard.
Keeping the headline number in sync with the artifacts by hand is how READMEs end
up lying. Run this after ``gridpulse train``::
python scripts/update_readme.py
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
README = REPO_ROOT / "README.md"
LEADERBOARD = REPO_ROOT / "artifacts" / "leaderboard.json"
HEADLINE = REPO_ROOT / "artifacts" / "headline.json"
START = "<!-- RESULTS:START -->"
END = "<!-- RESULTS:END -->"
PRETTY = {
"gbm": "**LightGBM** (global, quantile)",
"gbm_hybrid": "**LightGBM hybrid** (+ EIA forecast as input)",
"lstm": "**LSTM** encoder",
"transformer": "**Transformer** encoder",
"ensemble": "**Ensemble** (GBM + LSTM)",
"eia_official": "_EIA official forecast_ ⭐",
"seasonal_naive": "Seasonal naive (24h)",
"weekly_naive": "Weekly naive (168h)",
"drift_naive": "Drift naive",
}
QUANTILE_MODELS = {"gbm_p10", "gbm_p50", "gbm_p90"}
def render_table(rows: list[dict]) -> str:
lines = [
"| Model | MAPE % | MAE (MW) | RMSE (MW) | R² | Peak-hour MAPE % | Skill vs EIA |",
"|---|---|---|---|---|---|---|",
]
for row in sorted(rows, key=lambda r: r.get("mape_pct", 999)):
if row["model"] in QUANTILE_MODELS:
continue
name = PRETTY.get(row["model"], row["model"])
skill = row.get("skill_vs_eia_pct")
if row["model"] == "eia_official":
skill_cell = "- (benchmark)"
elif isinstance(skill, int | float):
skill_cell = f"**{skill:+.1f}%**" if skill > 0 else f"{skill:+.1f}%"
else:
skill_cell = "-"
lines.append(
f"| {name} | {row['mape_pct']:.3f} | {row['mae_mwh']:,.0f} | "
f"{row['rmse_mwh']:,.0f} | {row['r2']:.4f} | "
f"{row.get('peak_hour_mape_pct', float('nan')):.3f} | {skill_cell} |"
)
return "\n".join(lines)
def main() -> int:
if not LEADERBOARD.exists():
print(f"No leaderboard at {LEADERBOARD}. Run `gridpulse train` first.", file=sys.stderr)
return 1
rows = json.loads(LEADERBOARD.read_text())
table = render_table(rows)
if HEADLINE.exists():
head = json.loads(HEADLINE.read_text())
skill = head.get("skill_vs_eia_pct")
if isinstance(skill, int | float) and skill > 0:
label = PRETTY.get(head["best_model"], head["best_model"]).replace("*", "")
table = (
f"> ### {skill:.1f}% more accurate than the EIA's own day-ahead forecast\n"
f">\n"
f"> **{label}** reaches **{head['best_mape_pct']:.3f}% MAPE** against the "
f"EIA's **{head['eia_benchmark_mape_pct']:.3f}%**, measured over "
f"**{head['test_observations']:,}** out-of-sample hours across 12 balancing "
f"authorities.\n"
f">\n"
f"> Trained without ever seeing the test window. The EIA benchmark is the "
f"forecast the US government actually published and grid operators actually "
f"operated against.\n\n" + table
)
table += (
"\n\n<sub>P10/P50/P90 quantile models are omitted above: they define the "
"prediction interval rather than competing as point forecasts. Interval "
"calibration is reported separately.</sub>"
)
text = README.read_text(encoding="utf-8")
if START not in text or END not in text:
print("README is missing the RESULTS markers.", file=sys.stderr)
return 1
before = text.split(START)[0]
after = text.split(END)[1]
README.write_text(f"{before}{START}\n{table}\n{END}{after}", encoding="utf-8")
print("README results table updated.")
print(table)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|