| from __future__ import annotations
|
|
|
| from functools import lru_cache
|
| from pathlib import Path
|
|
|
| import joblib
|
| import lightgbm as lgb
|
| import pandas as pd
|
| import plotly.express as px
|
| import plotly.graph_objects as go
|
|
|
| from dashboard.runtime import configure_runtime
|
|
|
| configure_runtime()
|
|
|
| from huggingface_hub import hf_hub_download
|
|
|
| REGIONS = ["NSW1", "VIC1", "QLD1", "SA1", "TAS1"]
|
| HORIZONS = {
|
| "5 minutes (1 step)": 1,
|
| "30 minutes (6 steps)": 6,
|
| "1 hour (12 steps)": 12,
|
| }
|
| TIMESTAMP_COLUMN = "SETTLEMENTDATE"
|
| REGION_COLUMN = "REGIONID"
|
| DEMAND_COLUMN = "TOTALDEMAND"
|
| DATASET_REPO = "gausubash/nem-demand-5min"
|
| FALLBACK_DATASET_REPO = "PowerZooJax/PowerZooDataset"
|
| FALLBACK_DATASET_FILE = "parquet/AEMO_5min_Demand_2025_2026.parquet"
|
| ASSETS_DIR = Path(__file__).resolve().parent.parent / "assets"
|
| LOCAL_DEMAND_PATH = ASSETS_DIR / "demand_all_regions.parquet"
|
|
|
|
|
| @lru_cache(maxsize=1)
|
| def load_demand() -> pd.DataFrame:
|
| if LOCAL_DEMAND_PATH.exists():
|
| frame = pd.read_parquet(LOCAL_DEMAND_PATH)
|
| else:
|
| download_dir = ASSETS_DIR / "downloads"
|
| download_dir.mkdir(parents=True, exist_ok=True)
|
| try:
|
| path = hf_hub_download(
|
| repo_id=DATASET_REPO,
|
| repo_type="dataset",
|
| filename="demand_all_regions.parquet",
|
| local_dir=str(download_dir),
|
| local_dir_use_symlinks=False,
|
| )
|
| frame = pd.read_parquet(path)
|
| except Exception:
|
| path = hf_hub_download(
|
| repo_id=FALLBACK_DATASET_REPO,
|
| repo_type="dataset",
|
| filename=FALLBACK_DATASET_FILE,
|
| local_dir=str(download_dir),
|
| local_dir_use_symlinks=False,
|
| )
|
| raw = pd.read_parquet(path)
|
| frame = raw.rename(
|
| columns={
|
| "INTERVAL_DATETIME": TIMESTAMP_COLUMN,
|
| "OPERATIONAL_DEMAND": DEMAND_COLUMN,
|
| }
|
| )
|
| frame[TIMESTAMP_COLUMN] = pd.to_datetime(frame[TIMESTAMP_COLUMN], utc=True).dt.tz_localize(None)
|
|
|
| frame[TIMESTAMP_COLUMN] = pd.to_datetime(frame[TIMESTAMP_COLUMN])
|
| frame[DEMAND_COLUMN] = frame[DEMAND_COLUMN].astype(float)
|
| return frame.sort_values([REGION_COLUMN, TIMESTAMP_COLUMN]).reset_index(drop=True)
|
|
|
|
|
| @lru_cache(maxsize=1)
|
| def load_benchmark_metrics() -> pd.DataFrame:
|
| path = ASSETS_DIR / "benchmark_metrics.csv"
|
| if path.exists():
|
| return pd.read_csv(path)
|
| return pd.DataFrame()
|
|
|
|
|
| def load_lightgbm_model(region: str, horizon: int) -> lgb.LGBMRegressor | None:
|
| path = ASSETS_DIR / "models" / f"{region}_h{horizon}.pkl"
|
| if not path.exists():
|
| return None
|
| return joblib.load(path)
|
|
|
|
|
| def horizon_label(steps: int) -> str:
|
| for label, value in HORIZONS.items():
|
| if value == steps:
|
| return label
|
| return f"{steps} steps"
|
|
|
|
|
| def demand_summary_table(demand: pd.DataFrame) -> pd.DataFrame:
|
| rows = []
|
| for region in REGIONS:
|
| region_frame = demand.loc[demand[REGION_COLUMN] == region]
|
| if region_frame.empty:
|
| continue
|
| rows.append(
|
| {
|
| "Region": region,
|
| "Rows": len(region_frame),
|
| "Start": region_frame[TIMESTAMP_COLUMN].min(),
|
| "End": region_frame[TIMESTAMP_COLUMN].max(),
|
| "Min MW": round(region_frame[DEMAND_COLUMN].min(), 1),
|
| "Max MW": round(region_frame[DEMAND_COLUMN].max(), 1),
|
| "Mean MW": round(region_frame[DEMAND_COLUMN].mean(), 1),
|
| }
|
| )
|
| return pd.DataFrame(rows)
|
|
|
|
|
| def region_timeseries_figure(demand: pd.DataFrame, region: str, days: int = 7) -> go.Figure:
|
| region_frame = demand.loc[demand[REGION_COLUMN] == region].copy()
|
| cutoff = region_frame[TIMESTAMP_COLUMN].max() - pd.Timedelta(days=days)
|
| region_frame = region_frame.loc[region_frame[TIMESTAMP_COLUMN] >= cutoff]
|
| fig = px.line(
|
| region_frame,
|
| x=TIMESTAMP_COLUMN,
|
| y=DEMAND_COLUMN,
|
| title=f"{region} demand — last {days} days",
|
| labels={DEMAND_COLUMN: "Demand (MW)", TIMESTAMP_COLUMN: "Time"},
|
| )
|
| fig.update_layout(template="plotly_dark", height=420, margin=dict(l=40, r=20, t=60, b=40))
|
| return fig
|
|
|
|
|
| def benchmark_bar_figure(metrics: pd.DataFrame, region: str, horizon_steps: int) -> go.Figure:
|
| subset = metrics.loc[
|
| (metrics["region"] == region) & (metrics["horizon_steps"] == horizon_steps)
|
| ].copy()
|
| if subset.empty:
|
| fig = go.Figure()
|
| fig.update_layout(title="Benchmark metrics not available yet", template="plotly_dark")
|
| return fig
|
|
|
| subset = subset.sort_values("mae")
|
| fig = px.bar(
|
| subset,
|
| x="model",
|
| y="mae",
|
| title=f"Model comparison — {region}, {horizon_label(horizon_steps)}",
|
| labels={"mae": "MAE (MW)", "model": "Model"},
|
| color="model",
|
| )
|
| fig.update_layout(template="plotly_dark", height=420, showlegend=False)
|
| return fig
|
|
|
|
|
| def benchmark_heatmap_figure(metrics: pd.DataFrame, model: str) -> go.Figure:
|
| subset = metrics.loc[metrics["model"] == model].copy()
|
| if subset.empty:
|
| fig = go.Figure()
|
| fig.update_layout(title=f"No metrics for {model}", template="plotly_dark")
|
| return fig
|
|
|
| pivot = subset.pivot(index="region", columns="horizon_steps", values="mae")
|
| fig = px.imshow(
|
| pivot,
|
| text_auto=".1f",
|
| aspect="auto",
|
| title=f"MAE heatmap — {model}",
|
| labels={"color": "MAE (MW)"},
|
| )
|
| fig.update_layout(template="plotly_dark", height=420)
|
| return fig
|
|
|
|
|
| def forecast_with_chronos(
|
| demand: pd.DataFrame,
|
| region: str,
|
| anchor_time: pd.Timestamp,
|
| horizon: int,
|
| ) -> tuple[go.Figure, pd.DataFrame]:
|
| from dashboard.chronos_model import predict_demand
|
|
|
| region_frame = demand.loc[demand[REGION_COLUMN] == region].sort_values(TIMESTAMP_COLUMN)
|
| history = region_frame.loc[region_frame[TIMESTAMP_COLUMN] <= anchor_time].tail(512)
|
| if len(history) < 24:
|
| raise ValueError("Not enough history before the selected timestamp.")
|
|
|
| context_df = pd.DataFrame(
|
| {
|
| "item_id": region,
|
| "timestamp": history[TIMESTAMP_COLUMN],
|
| "target": history[DEMAND_COLUMN],
|
| }
|
| )
|
| pred_df = predict_demand(context_df, horizon)
|
|
|
| history_plot = history.tail(288)
|
| fig = go.Figure()
|
| fig.add_trace(
|
| go.Scatter(
|
| x=history_plot[TIMESTAMP_COLUMN],
|
| y=history_plot[DEMAND_COLUMN],
|
| mode="lines",
|
| name="History",
|
| line=dict(color="#60a5fa"),
|
| )
|
| )
|
| fig.add_trace(
|
| go.Scatter(
|
| x=pred_df["timestamp"],
|
| y=pred_df["0.5"],
|
| mode="lines+markers",
|
| name="Forecast (p50)",
|
| line=dict(color="#34d399"),
|
| )
|
| )
|
| fig.add_trace(
|
| go.Scatter(
|
| x=pred_df["timestamp"],
|
| y=pred_df["0.9"],
|
| mode="lines",
|
| line=dict(width=0),
|
| showlegend=False,
|
| )
|
| )
|
| fig.add_trace(
|
| go.Scatter(
|
| x=pred_df["timestamp"],
|
| y=pred_df["0.1"],
|
| mode="lines",
|
| fill="tonexty",
|
| name="80% interval",
|
| line=dict(width=0),
|
| fillcolor="rgba(52, 211, 153, 0.2)",
|
| )
|
| )
|
| fig.update_layout(
|
| title=f"Chronos GPU forecast — {region}, {horizon_label(horizon)}",
|
| template="plotly_dark",
|
| height=460,
|
| xaxis_title="Time",
|
| yaxis_title="Demand (MW)",
|
| )
|
| display_df = pred_df[["timestamp", "0.1", "0.5", "0.9"]].rename(
|
| columns={"0.1": "p10", "0.5": "p50", "0.9": "p90"}
|
| )
|
| return fig, display_df
|
|
|
