Sync from GitHub 1db426b

app.py

@@ -1,10 +1,5 @@
-"""GridPulse public web application.
-
-Runs locally or on Streamlit Community Cloud from the same file. The app reads the
-slim DuckDB artifact committed alongside it and the pre-trained model files, so it
-starts instantly and never trains on the request path. Live weather is fetched on
-demand to produce genuinely forward-looking forecasts.
-"""
+"""The public GridPulse website. Reads the committed database and model files, so
+it starts instantly and never trains anything while someone is waiting."""
 
 from __future__ import annotations
 
@@ -23,9 +18,6 @@ sys.path.insert(0, str(ROOT / "src"))
 from gridpulse.config import BALANCING_AUTHORITIES  # noqa: E402
 from gridpulse.warehouse.duck import connect  # noqa: E402
 
-# ---------------------------------------------------------------------------
-# Page setup
-# ---------------------------------------------------------------------------
 st.set_page_config(
     page_title="GridPulse | US Electricity Demand Intelligence",
     page_icon="⚡",
@@ -61,9 +53,6 @@ st.markdown(
 )
 
 
-# ---------------------------------------------------------------------------
-# Data access
-# ---------------------------------------------------------------------------
 def database_path() -> Path:
     slim = ROOT / "data" / "gold" / "gridpulse_app.duckdb"
     return slim if slim.exists() else ROOT / "data" / "gold" / "gridpulse.duckdb"
@@ -161,9 +150,6 @@ def data_ready() -> bool:
     ).empty
 
 
-# ---------------------------------------------------------------------------
-# Header
-# ---------------------------------------------------------------------------
 head = headline()
 skill = head.get("skill_vs_eia_pct")
 
@@ -192,9 +178,6 @@ if not data_ready():
     )
     st.stop()
 
-# ---------------------------------------------------------------------------
-# Sidebar
-# ---------------------------------------------------------------------------
 with st.sidebar:
     st.header("Controls")
     bas = available_bas()
@@ -224,9 +207,6 @@ with st.sidebar:
         "[GitHub repository](https://github.com/adwitiyashukla/gridpulse) · Data: US EIA + Open-Meteo"
     )
 
-# ---------------------------------------------------------------------------
-# Headline metrics
-# ---------------------------------------------------------------------------
 c1, c2, c3, c4 = st.columns(4)
 summary = run_query(
     """
@@ -252,9 +232,6 @@ tabs = st.tabs([
 ])
 
 
-# ---------------------------------------------------------------------------
-# Tab 1: Forecast
-# ---------------------------------------------------------------------------
 with tabs[0]:
     st.subheader(f"24-hour demand forecast - {selected_ba}")
     st.caption(
@@ -292,9 +269,6 @@ with tabs[0]:
         for note in notes:
             st.caption(f"↳ {note}")
 
-        # demand_clean_mwh excludes readings the warehouse flagged as physically
-        # implausible. The raw column deliberately retains them as evidence, but a
-        # telemetry fault should not be drawn as though it were real load.
         history = run_query(
             """
             SELECT period_utc, demand_clean_mwh AS demand_mwh
@@ -355,9 +329,6 @@ with tabs[0]:
         st.info("Choose a balancing authority in the sidebar and press **Generate forecast**.")
 
 
-# ---------------------------------------------------------------------------
-# Tab 2: Explorer
-# ---------------------------------------------------------------------------
 with tabs[1]:
     st.subheader(f"Historical explorer - {selected_ba}")
 
@@ -402,9 +373,6 @@ with tabs[1]:
                     labels={"temperature_2m": "Temperature (°C)", "demand_mwh": "Demand (MW)"},
                 )
 
-                # Binned median rather than a lowess fit. It needs no extra
-                # dependency, is robust to the outliers this data genuinely
-                # contains, and traces the V-curve more legibly than a smoother.
                 binned = (
                     scatter.assign(bin=(scatter["temperature_2m"] / 2).round() * 2)
                     .groupby("bin")["demand_mwh"]
@@ -465,9 +433,6 @@ with tabs[1]:
             st.plotly_chart(figure, use_container_width=True)
 
 
-# ---------------------------------------------------------------------------
-# Tab 3: Leaderboard
-# ---------------------------------------------------------------------------
 with tabs[2]:
     st.subheader("Model leaderboard")
     st.caption(
@@ -495,10 +460,6 @@ with tabs[2]:
         }
         board["Model"] = board["model"].map(lambda m: pretty.get(m, m))
 
-        # Sorted worst-first so the best model lands at the top of a horizontal
-        # bar chart. Colour is applied per-bar via marker_color rather than
-        # plotly's `color=` argument: that argument splits the data into one
-        # trace per colour group, which silently destroys the sort order.
         ordered = board.sort_values("mape_pct", ascending=False)
         bar_colours = [
             ACCENT_2 if model == "eia_official" else ACCENT
@@ -564,9 +525,6 @@ with tabs[2]:
             st.plotly_chart(figure, use_container_width=True)
 
 
-# ---------------------------------------------------------------------------
-# Tab 4: Anomalies
-# ---------------------------------------------------------------------------
 with tabs[3]:
     st.subheader("Anomaly monitor")
     st.caption(
@@ -611,9 +569,6 @@ with tabs[3]:
         st.dataframe(recent, use_container_width=True, hide_index=True, height=380)
 
 
-# ---------------------------------------------------------------------------
-# Tab 5: Data quality
-# ---------------------------------------------------------------------------
 with tabs[4]:
     st.subheader("Data quality scorecard")
     st.caption(
@@ -660,9 +615,6 @@ with tabs[4]:
         )
 
 
-# ---------------------------------------------------------------------------
-# Tab 6: AI agent
-# ---------------------------------------------------------------------------
 with tabs[5]:
     st.subheader("Ask the Grid")
     st.caption(
@@ -707,7 +659,6 @@ with tabs[5]:
 
                 st.dataframe(answer.data, use_container_width=True, hide_index=True, height=380)
 
-                # Offer a chart when the shape obviously supports one.
                 numeric = answer.data.select_dtypes("number").columns.tolist()
                 if len(answer.data) > 1 and numeric:
                     label_columns = [c for c in answer.data.columns if c not in numeric]
@@ -724,9 +675,6 @@ with tabs[5]:
                             pass
 
 
-# ---------------------------------------------------------------------------
-# Tab 7: How it works
-# ---------------------------------------------------------------------------
 with tabs[6]:
     st.subheader("How GridPulse works")
 

orchestration/airflow/dags/gridpulse_daily.py

@@ -40,7 +40,6 @@ DEFAULT_ARGS = {
     dag_id="gridpulse_daily_refresh",
     description="Extract EIA and weather, rebuild the warehouse, validate, retrain, export.",
     default_args=DEFAULT_ARGS,
-    # 06:00 UTC: EIA has published the previous full day by then.
     schedule="0 6 * * *",
     start_date=datetime(2024, 1, 1),
     catchup=False,

orchestration/dagster_app/definitions.py

@@ -1,24 +1,10 @@
-"""The GridPulse pipeline written as Dagster assets.
+"""The GridPulse pipeline as Dagster assets. Run with ``dagster dev -f`` this file.
 
-Dagster is built around *assets*, meaning the tables and files that need to exist,
-rather than around *tasks* that need to run. That fits this project well, because
-every node below really is a table or a file. So the diagram in the Dagster UI is
-an actual picture of the warehouse, not just a schedule.
-
-To run the UI locally::
-
-    dagster dev -f orchestration/dagster_app/definitions.py
-
-There is an equivalent Airflow DAG in ``orchestration/airflow/dags/`` for anyone
-whose team uses Airflow instead.
+Do not add ``from __future__ import annotations`` here. It turns annotations into
+strings, and Dagster reads the annotation on ``context`` to decide what to inject,
+so the assets fail to load with a misleading error.
 """
 
-# NOTE: do not add `from __future__ import annotations` to this file.
-# It turns every type annotation into a string at runtime, and Dagster reads the
-# annotation on the `context` argument to work out what to pass in. With that
-# import it sees the text "AssetExecutionContext" instead of the actual class and
-# throws DagsterInvalidDefinitionError, with an error message that confusingly
-# names the exact type you already gave it.
 
 import sys
 from pathlib import Path
@@ -46,9 +32,6 @@ GROUP_ML = "04_machine_learning"
 GROUP_SERVE = "05_serving"
 
 
-# ---------------------------------------------------------------------------
-# Extract
-# ---------------------------------------------------------------------------
 @asset(
     group_name=GROUP_EXTRACT,
     compute_kind="python",
@@ -89,9 +72,6 @@ def weather_bronze(context: AssetExecutionContext) -> Output[dict]:
     )
 
 
-# ---------------------------------------------------------------------------
-# Warehouse
-# ---------------------------------------------------------------------------
 @asset(
     group_name=GROUP_WAREHOUSE,
     compute_kind="duckdb",
@@ -139,9 +119,6 @@ def dbt_marts(context: AssetExecutionContext) -> Output[str]:
     )
 
 
-# ---------------------------------------------------------------------------
-# Quality
-# ---------------------------------------------------------------------------
 @asset(
     group_name=GROUP_QUALITY,
     compute_kind="python",
@@ -193,9 +170,6 @@ def benchmark_present() -> AssetCheckResult:
     return AssetCheckResult(passed=rows > 1000, metadata={"benchmark_rows": rows})
 
 
-# ---------------------------------------------------------------------------
-# Machine learning
-# ---------------------------------------------------------------------------
 @asset(
     group_name=GROUP_ML,
     compute_kind="python",
@@ -247,9 +221,6 @@ def anomaly_scores(context: AssetExecutionContext) -> Output[dict]:
     )
 
 
-# ---------------------------------------------------------------------------
-# Serving
-# ---------------------------------------------------------------------------
 @asset(
     group_name=GROUP_SERVE,
     compute_kind="python",
@@ -271,9 +242,6 @@ def app_export(context: AssetExecutionContext) -> Output[str]:
     )
 
 
-# ---------------------------------------------------------------------------
-# Jobs and schedules
-# ---------------------------------------------------------------------------
 daily_refresh = define_asset_job(
     name="daily_refresh",
     selection=AssetSelection.all(),
@@ -286,7 +254,6 @@ incremental_refresh = define_asset_job(
     description="Data-only refresh without retraining. Cheap enough to run hourly.",
 )
 
-# EIA publishes on a lag, so 06:00 UTC comfortably captures the previous full day.
 daily_schedule = ScheduleDefinition(
     job=daily_refresh,
     cron_schedule="0 6 * * *",
@@ -294,10 +261,6 @@ daily_schedule = ScheduleDefinition(
     description="Nightly full refresh after EIA publishes the previous day.",
 )
 
-# Left at Dagster's default status (stopped), so it appears in the UI and can be
-# enabled deliberately rather than starting to poll the moment anyone runs the
-# project. Passing default_status=None is not the way to express that: the
-# parameter expects a DefaultScheduleStatus enum and rejects None outright.
 hourly_schedule = ScheduleDefinition(
     job=incremental_refresh,
     cron_schedule="15 * * * *",

scripts/update_readme.py

@@ -33,8 +33,6 @@ PRETTY = {
 }
 
 
-# P10/P50/P90 are the edges of a prediction interval, not competing point
-# forecasts. Ranking them by MAPE compares things that answer different questions.
 QUANTILE_MODELS = {"gbm_p10", "gbm_p50", "gbm_p90"}
 
 

src/gridpulse/agent/text2sql.py

@@ -1,29 +1,8 @@
-"""Letting people ask questions in plain English instead of writing SQL.
-
-You can ask "which region had the worst forecast error last month?" and get a
-chart back rather than a SQL editor. The part I actually had to think about is not
-calling the LLM, which is about four lines of code. It is not trusting what comes
-back from it.
-
-There are six checks, in this order:
-
-1. **The connection is read-only.** I open the database with ``read_only=True``,
-   so even if someone completely tricked the model, it still could not change
-   anything.
-2. **Only one statement, and it has to be a SELECT.** It must start with
-   ``SELECT`` or ``WITH``. Anything else gets rejected before it runs.
-3. **Banned keywords.** Anything that creates, changes or deletes data is refused,
-   including when it is hidden inside a comment or tacked on after a semicolon.
-4. **A list of allowed tables.** Only my actual warehouse tables can be queried,
-   which stops anyone reading DuckDB's internal system tables.
-5. **A forced LIMIT.** If the model forgets one, I add it, so no single query can
-   return a huge amount of data.
-6. **The real schema goes into the prompt.** I read the actual table and column
-   names out of the database and put them in the prompt, so the model describes
-   real columns instead of inventing ones that sound plausible.
-
-The SQL it generated always comes back along with the answer. If you cannot see
-the query, you have no way to judge whether the answer is right.
+"""Turns plain English questions into DuckDB SQL, behind six safety checks.
+
+Read-only connection, one statement only, SELECT or WITH only, a banned keyword
+list, a table allowlist, and a forced LIMIT. The generated SQL is always returned
+alongside the answer.
 """
 
 from __future__ import annotations
@@ -106,9 +85,6 @@ class AgentAnswer:
         return self.error is None
 
 
-# ---------------------------------------------------------------------------
-# Guard
-# ---------------------------------------------------------------------------
 def _strip_fences(text: str) -> str:
     text = re.sub(r"^\s*```(?:sql)?\s*", "", text.strip(), flags=re.IGNORECASE)
     return re.sub(r"\s*```\s*$", "", text).strip()
@@ -145,7 +121,6 @@ def guard_sql(raw: str, allowed_tables: set[str] | None = None) -> str:
         match.lower()
         for match in re.findall(r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)", body, flags=re.IGNORECASE)
     }
-    # CTE names are defined inline and are legitimate targets.
     cte_names = {m.lower() for m in re.findall(r"(?:with|,)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+as\s*\(", body, flags=re.IGNORECASE)}
     unknown = referenced - allowed - cte_names
     if unknown:
@@ -160,9 +135,6 @@ def guard_sql(raw: str, allowed_tables: set[str] | None = None) -> str:
     return body
 
 
-# ---------------------------------------------------------------------------
-# Schema grounding
-# ---------------------------------------------------------------------------
 def introspect_schema(database=None, tables: set[str] | None = None) -> str:
     """Render a compact schema description for the prompt."""
     wanted = tables or ALLOWED_TABLES
@@ -184,9 +156,6 @@ def introspect_schema(database=None, tables: set[str] | None = None) -> str:
     return "\n".join(lines) if lines else "(warehouse is empty)"
 
 
-# ---------------------------------------------------------------------------
-# Agent
-# ---------------------------------------------------------------------------
 class GridAgent:
     """Question in, validated SQL and a DataFrame out."""
 
@@ -270,7 +239,6 @@ class GridAgent:
             try:
                 sql = guard_sql(raw)
             except SQLGuardError as first_failure:
-                # One repair attempt: hand the model its own error and ask again.
                 warnings.append(f"First attempt rejected: {first_failure}")
                 logger.info("Guard rejected SQL, retrying: %s", first_failure)
                 repair = self._get_client().chat.completions.create(

src/gridpulse/api/main.py

@@ -1,13 +1,4 @@
-"""GridPulse REST API.
-
-A thin, documented HTTP surface over the warehouse, the trained models and the
-analytics agent. FastAPI generates OpenAPI docs at ``/docs`` automatically, so the
-service is self-describing.
-
-Run locally::
-
-    uvicorn gridpulse.api.main:app --reload --port 8000
-"""
+"""REST API over the warehouse and models. Docs at /docs when the server runs."""
 
 from __future__ import annotations
 
@@ -55,9 +46,6 @@ def _query(sql: str, params: list | None = None):
         raise HTTPException(status_code=503, detail=f"Warehouse unavailable: {exc}") from exc
 
 
-# ---------------------------------------------------------------------------
-# Schemas
-# ---------------------------------------------------------------------------
 class HealthResponse(BaseModel):
     status: str
     warehouse_present: bool
@@ -77,9 +65,6 @@ class AskRequest(BaseModel):
     summarise: bool = True
 
 
-# ---------------------------------------------------------------------------
-# Routes
-# ---------------------------------------------------------------------------
 @app.get("/", include_in_schema=False)
 def root() -> dict[str, Any]:
     return {"service": "GridPulse API", "version": "1.0.0", "docs": "/docs"}

src/gridpulse/cli.py

@@ -1,16 +1,5 @@
-"""GridPulse command line interface.
-
-Every stage of the platform is reachable from one entry point::
-
-    gridpulse probe            # validate API credentials and response contracts
-    gridpulse ingest           # extract EIA + weather into bronze
-    gridpulse build            # bronze -> silver -> gold warehouse
-    gridpulse quality          # run the data quality suite
-    gridpulse train            # train and evaluate the full model suite
-    gridpulse anomalies        # fit and score anomaly detectors
-    gridpulse export           # write deployment artifacts for the public app
-    gridpulse all              # the entire pipeline, in order
-"""
+"""One command for every stage: probe, ingest, build, quality, train, anomalies,
+export, or all of them in order."""
 
 from __future__ import annotations
 
@@ -44,9 +33,6 @@ def _timed(label: str, fn: Callable, *args, **kwargs):
     return result
 
 
-# ---------------------------------------------------------------------------
-# Commands
-# ---------------------------------------------------------------------------
 def cmd_probe(args: argparse.Namespace) -> int:
     from gridpulse.config import SETTINGS, active_bas
     from gridpulse.ingestion import probe_eia
@@ -121,15 +107,12 @@ def cmd_export(args: argparse.Namespace) -> int:
 def cmd_all(args: argparse.Namespace) -> int:
     for step in (cmd_ingest, cmd_build, cmd_quality, cmd_train, cmd_anomalies, cmd_export):
         code = step(args)
-        if code != 0 and step is not cmd_quality:  # quality warnings should not halt the run
+        if code != 0 and step is not cmd_quality:
             return code
     _banner("PIPELINE COMPLETE")
     return 0
 
 
-# ---------------------------------------------------------------------------
-# Parser
-# ---------------------------------------------------------------------------
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="gridpulse",
@@ -183,7 +166,6 @@ def main(argv: list[str] | None = None) -> int:
     for flag in ("full_refresh", "rebuild", "quick", "eia_only", "weather_only"):
         setattr(args, flag, getattr(args, flag, False))
 
-    # Normalise the comma-separated BA filter into a list once, here.
     args.bas = [c.strip().upper() for c in args.bas.split(",") if c.strip()] if args.bas else None
     try:
         return args.func(args)

src/gridpulse/config.py

@@ -1,8 +1,4 @@
-"""Central configuration and the balancing-authority registry.
-
-Every path in the project resolves from :data:`PATHS`, so the same code runs
-identically on a laptop, in CI and on Streamlit Community Cloud.
-"""
+"""Settings, file paths and the list of 12 balancing authorities."""
 
 from __future__ import annotations
 
@@ -10,7 +6,7 @@ import os
 from dataclasses import dataclass, field
 from pathlib import Path
 
-try:  # optional: absent on minimal app deployments
+try:
     from dotenv import load_dotenv
 
     load_dotenv()
@@ -31,17 +27,10 @@ def _resolve(p: str) -> Path:
     return path if path.is_absolute() else REPO_ROOT / path
 
 
-# ---------------------------------------------------------------------------
-# Balancing authority registry
-# ---------------------------------------------------------------------------
 @dataclass(frozen=True)
 class BalancingAuthority:
-    """A US balancing authority and the weather station proxy for its load centre.
-
-    ``latitude``/``longitude`` point at the dominant population centre inside the
-    BA footprint. Electricity demand is overwhelmingly driven by weather where the
-    people are, not by the geographic centroid of the territory.
-    """
+    """A region, with coordinates pointing at its biggest city rather than its
+    geographic centre, since demand follows the weather where people live."""
 
     code: str
     name: str
@@ -87,14 +76,11 @@ def active_bas() -> list[BalancingAuthority]:
     return [BALANCING_AUTHORITIES[c] for c in requested]
 
 
-# ---------------------------------------------------------------------------
-# Measure registry (EIA-930 `type` facet on the region-data endpoint)
-# ---------------------------------------------------------------------------
 EIA_MEASURES: dict[str, str] = {
-    "D":  "demand_mwh",             # actual hourly demand
-    "DF": "demand_forecast_mwh",    # EIA's OWN published day-ahead forecast -> our benchmark
-    "NG": "net_generation_mwh",     # net generation
-    "TI": "total_interchange_mwh",  # net exports to neighbours
+    "D":  "demand_mwh",
+    "DF": "demand_forecast_mwh",
+    "NG": "net_generation_mwh",
+    "TI": "total_interchange_mwh",
 }
 
 WEATHER_VARIABLES: list[str] = [
@@ -108,9 +94,6 @@ WEATHER_VARIABLES: list[str] = [
 ]
 
 
-# ---------------------------------------------------------------------------
-# Paths
-# ---------------------------------------------------------------------------
 @dataclass(frozen=True)
 class Paths:
     data: Path = field(default_factory=lambda: _resolve(_env("GRIDPULSE_DATA_DIR", "data")))
@@ -143,9 +126,6 @@ class Paths:
 PATHS = Paths()
 
 
-# ---------------------------------------------------------------------------
-# Runtime settings
-# ---------------------------------------------------------------------------
 @dataclass(frozen=True)
 class Settings:
     eia_api_key: str = field(default_factory=lambda: _env("EIA_API_KEY", ""))
@@ -171,7 +151,6 @@ class Settings:
 
 SETTINGS = Settings()
 
-# Horizon of the forecasting problem: predict the next 24 hours from the prior 7 days.
 FORECAST_HORIZON = 24
 LOOKBACK_HOURS = 168
 QUANTILES = (0.1, 0.5, 0.9)

src/gridpulse/features/build.py

@@ -1,33 +1,7 @@
-"""Building the features the models learn from.
-
-I picked these based on what actually makes electricity demand go up and down,
-rather than throwing every available column at the model:
-
-**Calendar features**
-    Hour, weekday and season are encoded as sine and cosine pairs, so that hour 23
-    sits right next to hour 0 instead of looking 23 units away from it. Holidays,
-    and the days either side of them, get their own flags, because office and
-    factory demand drops off a cliff on those days.
-
-**Lags**
-    Demand from 24, 48 and 168 hours ago. The 168 hour one (exactly a week) is the
-    strongest single predictor there is for this problem. Last Tuesday at 3pm looks
-    far more like this Tuesday at 3pm than 3am this morning does.
-
-**Rolling averages**
-    Averages and standard deviations over recent hours, all shifted back by the
-    full 24 hour forecast horizon, so nothing that would only be known after
-    prediction time can sneak in.
-
-**Weather**
-    Temperature, plus heating degrees and cooling degrees kept as separate columns.
-    Demand against temperature is V-shaped rather than a straight line, so splitting
-    it into the cold side and the hot side lets even a simple model pick it up, and
-    helps a tree model find the turning point faster.
-
-On leakage: every feature only uses information that existed before the moment
-being predicted. The one exception is the weather, and that is fine, because a real
-grid operator also has tomorrow's weather forecast in hand.
+"""Builds the 40 model features: calendar, lags, rolling stats and weather.
+
+Everything derived from past demand is shifted back by the full forecast horizon,
+so no feature can see anything that was not available at prediction time.
 """
 
 from __future__ import annotations
@@ -42,18 +16,8 @@ from gridpulse.warehouse.duck import query
 
 logger = logging.getLogger(__name__)
 
-# Comfort baseline in Celsius. Below it people heat, above it they cool.
 BALANCE_POINT_C = 18.0
 
-# Physically plausible band for demand, expressed as a multiple of each balancing
-# authority's own median. Real system load is remarkably well behaved: even the
-# most extreme heatwave peak sits under twice the annual median, and the deepest
-# overnight trough stays above a third of it. Anything outside this band is a
-# telemetry fault, not weather.
-#
-# This matters more than it looks. The raw EIA feed contains occasional readings
-# several orders of magnitude too large, and a handful of them inflated PJM's
-# standard deviation to 10.7 million MW against a true range near 70,000-165,000 MW.
 DEMAND_PLAUSIBLE_LOWER = 0.2
 DEMAND_PLAUSIBLE_UPPER = 5.0
 
@@ -61,22 +25,17 @@ LAG_HOURS = (24, 25, 26, 48, 72, 168, 336)
 ROLLING_WINDOWS = (24, 168)
 
 FEATURE_COLUMNS: list[str] = [
-    # cyclical calendar
     "hour_sin", "hour_cos", "dow_sin", "dow_cos", "doy_sin", "doy_cos",
-    # categorical calendar
     "is_weekend", "is_holiday", "is_business_day",
     "is_day_before_holiday", "is_day_after_holiday",
-    # autoregressive
     *[f"demand_lag_{h}h" for h in LAG_HOURS],
     *[f"demand_roll_mean_{w}h" for w in ROLLING_WINDOWS],
     *[f"demand_roll_std_{w}h" for w in ROLLING_WINDOWS],
     "demand_same_hour_last_week_delta",
-    # weather
     "temperature_2m", "apparent_temperature", "relative_humidity_2m",
     "dew_point_2m", "cloud_cover", "wind_speed_10m", "shortwave_radiation",
     "heating_degrees", "cooling_degrees", "temp_squared",
     "temp_lag_24h", "temp_change_24h", "temp_roll_mean_24h",
-    # interactions
     "cooling_x_business", "heating_x_business", "cooling_x_hour",
 ]
 
@@ -109,11 +68,7 @@ def load_modelling_frame(ba_codes: list[str] | None = None) -> pd.DataFrame:
 
 
 def flag_implausible_demand(frame: pd.DataFrame, target: str = TARGET) -> pd.Series:
-    """True where demand is impossible relative to the BA's own median.
-
-    Bounds are derived from the median rather than the mean precisely because the
-    outliers being hunted would drag a mean toward themselves and hide.
-    """
+    """True where demand is impossible compared to that region's own median."""
     median = frame.groupby("ba_code")[target].transform("median")
     return (frame[target] < median * DEMAND_PLAUSIBLE_LOWER) | (
         frame[target] > median * DEMAND_PLAUSIBLE_UPPER
@@ -130,7 +85,6 @@ def _engineer_one_ba(frame: pd.DataFrame) -> pd.DataFrame:
     """Build features for a single BA. Assumes the frame is sorted by period."""
     out = frame.sort_values("period_utc").copy()
 
-    # ---- calendar --------------------------------------------------------
     _cyclical(out, out["hour_local"], 24, "hour")
     _cyclical(out, out["day_of_week"], 7, "dow")
     _cyclical(out, pd.to_datetime(out["date_local"]).dt.dayofyear, 365.25, "doy")
@@ -139,24 +93,19 @@ def _engineer_one_ba(frame: pd.DataFrame) -> pd.DataFrame:
                  "is_day_before_holiday", "is_day_after_holiday"):
         out[flag] = out[flag].fillna(False).astype(int)
 
-    # ---- autoregressive --------------------------------------------------
     demand = out[TARGET]
     for lag in LAG_HOURS:
         out[f"demand_lag_{lag}h"] = demand.shift(lag)
 
-    # Rolling windows are shifted by the full horizon: at prediction time for
-    # hour t we only possess observations up to t - FORECAST_HORIZON.
     shifted = demand.shift(FORECAST_HORIZON)
     for window in ROLLING_WINDOWS:
         out[f"demand_roll_mean_{window}h"] = shifted.rolling(window, min_periods=window // 4).mean()
         out[f"demand_roll_std_{window}h"] = shifted.rolling(window, min_periods=window // 4).std()
 
-    # Week-on-week momentum at the same hour of day.
     out["demand_same_hour_last_week_delta"] = (
         out["demand_lag_168h"] - out["demand_lag_336h"]
     )
 
-    # ---- weather ---------------------------------------------------------
     temp = out["temperature_2m"]
     out["heating_degrees"] = (BALANCE_POINT_C - temp).clip(lower=0)
     out["cooling_degrees"] = (temp - BALANCE_POINT_C).clip(lower=0)
@@ -165,11 +114,8 @@ def _engineer_one_ba(frame: pd.DataFrame) -> pd.DataFrame:
     out["temp_change_24h"] = temp - out["temp_lag_24h"]
     out["temp_roll_mean_24h"] = temp.rolling(24, min_periods=6).mean()
 
-    # ---- interactions ----------------------------------------------------
-    # Offices and shops only run their air conditioning on working days.
     out["cooling_x_business"] = out["cooling_degrees"] * out["is_business_day"]
     out["heating_x_business"] = out["heating_degrees"] * out["is_business_day"]
-    # Afternoon heat compounds: the same 35C bites harder at 4pm than at 4am.
     out["cooling_x_hour"] = out["cooling_degrees"] * out["hour_local"]
 
     return out
@@ -180,17 +126,10 @@ def build_features(
     ba_codes: list[str] | None = None,
     dropna_target: bool = True,
 ) -> pd.DataFrame:
-    """Construct the full modelling matrix.
-
-    Parameters
-    ----------
-    frame
-        Pre-loaded gold data. Loaded from the warehouse when omitted.
-    ba_codes
-        Restrict to these balancing authorities.
-    dropna_target
-        Drop rows with no actual demand. Set False when building an inference
-        frame for future timestamps, where the target is legitimately unknown.
+    """Build the full feature table the models train on.
+
+    Set ``dropna_target=False`` when building features for future hours, where
+    there is no actual demand to compare against yet.
     """
     source = load_modelling_frame(ba_codes) if frame is None else frame
 
@@ -199,7 +138,6 @@ def build_features(
         ignore_index=True,
     )
 
-    # Weather can be sparse at the very edges of the archive/forecast seam.
     weather_columns = [
         "temperature_2m", "apparent_temperature", "relative_humidity_2m",
         "dew_point_2m", "cloud_cover", "wind_speed_10m", "shortwave_radiation",
@@ -209,9 +147,6 @@ def build_features(
         .transform(lambda s: s.interpolate(limit=6, limit_direction="both"))
     )
 
-    # Remove physically impossible readings before any statistic is computed from
-    # them. They are flagged in the warehouse and surfaced by the quality suite;
-    # here they are simply excluded from modelling.
     if dropna_target:
         implausible = flag_implausible_demand(engineered)
         if implausible.any():
@@ -223,7 +158,6 @@ def build_features(
         engineered = engineered[~implausible]
         engineered = engineered.dropna(subset=[TARGET])
 
-    # The deepest lag needs 336 hours of warm-up; those rows can never be complete.
     required = [c for c in FEATURE_COLUMNS if c.startswith("demand_lag")]
     engineered = engineered.dropna(subset=required)
 
@@ -237,11 +171,7 @@ def build_features(
 def chronological_split(
     frame: pd.DataFrame, test_days: int = 90, valid_days: int = 60
 ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-    """Split strictly by time, never randomly.
-
-    A random split on time-series data lets the model peek at the future through
-    neighbouring rows and produces gorgeous, meaningless validation scores.
-    """
+    """Split by date only, never randomly, so the model cannot see the future."""
     cutoff_test = frame["period_utc"].max() - pd.Timedelta(days=test_days)
     cutoff_valid = cutoff_test - pd.Timedelta(days=valid_days)
 

src/gridpulse/ingestion/eia.py

@@ -1,22 +1,7 @@
-"""Downloading the hourly grid data from the EIA API.
+"""Downloads hourly demand, EIA's own forecast, generation and interchange.
 
-For each region I pull four hourly series from the ``region-data`` endpoint:
-
-==========  ====================================================================
-``D``       Demand in MWh, which is what I am trying to predict
-``DF``      EIA's own day-ahead forecast, which is what I compare against
-``NG``      Net generation in MWh
-``TI``      Power traded with neighbouring regions, in MWh
-==========  ====================================================================
-
-The ``DF`` series is the important one. Instead of making up my own easy baseline,
-I score every model against the forecast the US government actually published and
-actually ran the grid against that day.
-
-Downloads only fetch what is new. Each run looks at the latest timestamp already
-saved and asks for periods after that, so a scheduled run costs a handful of
-requests rather than downloading everything again. Running it twice does not create
-duplicates.
+Only fetches periods newer than what is already saved, and running it twice does
+not create duplicate rows.
 """
 
 from __future__ import annotations
@@ -37,16 +22,12 @@ logger = logging.getLogger(__name__)
 EIA_BASE = "https://api.eia.gov/v2"
 REGION_DATA_ROUTE = f"{EIA_BASE}/electricity/rto/region-data/data/"
 
-# EIA-930 collection began 1 July 2015; requests before this return nothing.
 EIA_EPOCH = "2015-07-01"
 
 BRONZE_SUBDIR = "eia_region"
 _SCHEMA = ["period_utc", "ba_code", "measure_code", "value_mwh", "ingested_at_utc"]
 
 
-# ---------------------------------------------------------------------------
-# Bronze layout helpers
-# ---------------------------------------------------------------------------
 def bronze_path(ba_code: str) -> Path:
     return PATHS.bronze / BRONZE_SUBDIR / f"ba={ba_code}" / "data.parquet"
 
@@ -68,9 +49,6 @@ def watermark(ba_code: str) -> str | None:
     return latest.strftime("%Y-%m-%dT%H")
 
 
-# ---------------------------------------------------------------------------
-# Request construction
-# ---------------------------------------------------------------------------
 def _build_params(
     ba_code: str, start: str, end: str, offset: int, length: int
 ) -> list[tuple[str, str]]:
@@ -103,7 +81,6 @@ def _normalise(records: list[dict]) -> pd.DataFrame:
     df = pd.DataFrame(records)
     out = pd.DataFrame(
         {
-            # EIA hourly periods are UTC, formatted YYYY-MM-DDTHH
             "period_utc": pd.to_datetime(df["period"], format="%Y-%m-%dT%H", utc=True, errors="coerce"),
             "ba_code": df["respondent"].astype("string"),
             "measure_code": df["type"].astype("string"),
@@ -115,11 +92,7 @@ def _normalise(records: list[dict]) -> pd.DataFrame:
 
 
 def _merge(existing: pd.DataFrame, fresh: pd.DataFrame) -> pd.DataFrame:
-    """Merge new rows in. If a row already exists, the newest download wins.
-
-    Written this way so that running the download twice does not create duplicate
-    rows, which matters because the scheduled job can overlap with a manual run.
-    """
+    """Merge new rows in. If a row already exists, the newest download wins."""
     if existing.empty:
         combined = fresh
     elif fresh.empty:
@@ -140,9 +113,6 @@ def _merge(existing: pd.DataFrame, fresh: pd.DataFrame) -> pd.DataFrame:
     return combined
 
 
-# ---------------------------------------------------------------------------
-# Async fetch
-# ---------------------------------------------------------------------------
 async def _fetch_ba(
     client: httpx.AsyncClient,
     ba: BalancingAuthority,
@@ -200,7 +170,6 @@ async def _ingest_async(bas: list[BalancingAuthority], full_refresh: bool) -> di
         for ba in bas:
             mark = None if full_refresh else watermark(ba.code)
             if mark:
-                # Re-request the final stored day: EIA revises recent hours in place.
                 start_ts = pd.Timestamp(mark, tz="UTC") - pd.Timedelta(hours=24)
                 start = start_ts.strftime("%Y-%m-%dT%H")
             else:
@@ -218,9 +187,6 @@ async def _ingest_async(bas: list[BalancingAuthority], full_refresh: bool) -> di
     return written
 
 
-# ---------------------------------------------------------------------------
-# Public entry points
-# ---------------------------------------------------------------------------
 def ingest_eia(ba_codes: list[str] | None = None, full_refresh: bool = False) -> dict[str, int]:
     """Extract EIA-930 hourly telemetry into the bronze zone.
 

src/gridpulse/ingestion/http.py

@@ -1,9 +1,4 @@
-"""Shared async HTTP plumbing: bounded concurrency, retries and honest backoff.
-
-Public data APIs rate limit aggressively and fail transiently. Every network call
-in GridPulse funnels through :func:`fetch_json` so retry policy lives in exactly
-one place rather than being copy-pasted per source.
-"""
+"""Shared HTTP helper so the retry and backoff rules live in one place."""
 
 from __future__ import annotations
 
@@ -22,9 +17,6 @@ RATE_LIMIT_STATUS = 429
 MAX_ATTEMPTS = 6
 BASE_BACKOFF = 1.5
 
-# A rate limit is not a transient blip. Open-Meteo and similar public APIs meter
-# by weighted request cost over a rolling window, so the only useful response is
-# to wait meaningfully rather than retry three seconds later.
 RATE_LIMIT_BASE_WAIT = 20.0
 RATE_LIMIT_MAX_WAIT = 90.0
 

src/gridpulse/ingestion/weather.py

@@ -1,19 +1,7 @@
-"""Weather extraction from Open-Meteo (no API key required).
+"""Downloads weather from Open-Meteo, no API key needed.
 
-Weather is the dominant exogenous driver of electricity demand: air conditioning
-load in summer and resistive heating in winter produce the characteristic U-shaped
-demand-vs-temperature curve that every load forecaster models.
-
-Two endpoints are stitched together because neither alone is sufficient:
-
-* **ERA5 archive** -- authoritative reanalysis, but lags real time by about 5 days.
-* **Forecast endpoint** -- serves ``past_days`` of recent observations plus up to
-  16 days ahead, closing the archive gap and supplying the *future* covariates the
-  day-ahead model needs at inference time.
-
-Overlapping hours resolve in favour of the archive, which is the more accurate
-source. Getting this seam right is the difference between a model that works in a
-notebook and one that works tomorrow morning.
+Joins the ERA5 archive, which lags about 5 days, to the forecast endpoint, which
+covers the gap and supplies tomorrow's weather. Overlapping hours use the archive.
 """
 
 from __future__ import annotations
@@ -35,13 +23,10 @@ ARCHIVE_ROUTE = "https://archive-api.open-meteo.com/v1/archive"
 FORECAST_ROUTE = "https://api.open-meteo.com/v1/forecast"
 
 BRONZE_SUBDIR = "weather"
-ARCHIVE_LAG_DAYS = 6      # ERA5 publication delay, with a day of safety margin
-FORECAST_PAST_DAYS = 92   # maximum look-back the forecast endpoint allows
-FORECAST_AHEAD_DAYS = 16  # maximum look-ahead
+ARCHIVE_LAG_DAYS = 6
+FORECAST_PAST_DAYS = 92
+FORECAST_AHEAD_DAYS = 16
 
-# Open-Meteo's free tier meters weighted request cost over a rolling window.
-# A multi-year hourly archive pull is a heavy call, so requests are serialised
-# and spaced. This costs about a minute across all BAs and removes 429s entirely.
 INTER_REQUEST_PAUSE = 2.0
 
 
@@ -123,7 +108,6 @@ def _merge(*frames: pd.DataFrame) -> pd.DataFrame:
 
     combined = pd.concat(populated, ignore_index=True)
     combined["period_utc"] = pd.to_datetime(combined["period_utc"], utc=True)
-    # Rank so the archive sorts last and therefore wins `keep="last"`.
     combined["_priority"] = (combined["source"] == "era5_archive").astype(int)
     combined = (
         combined.sort_values(["period_utc", "_priority"])
@@ -136,7 +120,6 @@ def _merge(*frames: pd.DataFrame) -> pd.DataFrame:
 
 
 async def _ingest_async(bas: list[BalancingAuthority], full_refresh: bool) -> dict[str, int]:
-    # Serialised deliberately: see INTER_REQUEST_PAUSE above.
     sem = asyncio.Semaphore(1)
     today = datetime.now(timezone.utc).date()
     archive_end = today - timedelta(days=ARCHIVE_LAG_DAYS)
@@ -156,7 +139,6 @@ async def _ingest_async(bas: list[BalancingAuthority], full_refresh: bool) -> di
                     else SETTINGS.start_date
                 )
 
-            # Sequential rather than gathered, with a pause between calls.
             archive = await _fetch_archive(client, ba, archive_start, archive_end.isoformat(), sem)
             await asyncio.sleep(INTER_REQUEST_PAUSE)
             forecast = await _fetch_forecast(client, ba, sem)

src/gridpulse/models/anomaly.py

@@ -1,31 +1,5 @@
-"""Finding unusual hours: broken meters, sudden demand changes and strange days.
-
-I use three detectors and make them vote, because each one on its own misses a
-different kind of problem:
-
-**Seasonal z-score**
-    Compares each hour against the median demand for that region, at that hour of
-    the day, in that month, then scales by the median absolute deviation. I use
-    MAD rather than standard deviation because the weird values I am looking for
-    would inflate a standard deviation and end up hiding themselves. This catches
-    single spikes and drops, but it misses slow drift over several hours.
-
-**Isolation Forest**
-    Looks at several things at once: demand level, how fast demand is changing,
-    temperature, and how demand relates to temperature. This catches combinations
-    that look fine one at a time. Normal demand at a normal temperature can still
-    be odd if those two never usually go together.
-
-**Autoencoder on the daily shape**
-    A small neural network squeezes each day's 24 hour shape down to 8 numbers and
-    then tries to rebuild it. Days whose shape is unlike anything it trained on come
-    back badly rebuilt, even if every individual hour looks normal on its own. This
-    is the one that spots holidays acting like weekends, storm days, and days where
-    customers were paid to use less power.
-
-Making them vote keeps the false alarms down: an hour is only marked ``high``
-severity when at least two of the three detectors agree it is strange.
-"""
+"""Finds unusual hours using three detectors that have to agree: seasonal
+z-score, Isolation Forest and an autoencoder trained on daily demand shapes."""
 
 from __future__ import annotations
 
@@ -40,14 +14,11 @@ from gridpulse.warehouse.duck import connect, query
 
 logger = logging.getLogger(__name__)
 
-Z_THRESHOLD = 4.0          # robust z beyond which an hour is suspicious
-CONTAMINATION = 0.01       # expected anomaly rate for Isolation Forest
-AE_PERCENTILE = 99.0       # reconstruction-error percentile defining "unusual shape"
+Z_THRESHOLD = 4.0
+CONTAMINATION = 0.01
+AE_PERCENTILE = 99.0
 
 
-# ---------------------------------------------------------------------------
-# Detector 1: robust seasonal z-score
-# ---------------------------------------------------------------------------
 def robust_seasonal_z(frame: pd.DataFrame, target: str = "demand_mwh") -> pd.Series:
     """Median-absolute-deviation z-score within (BA, hour-of-day, month) cells."""
     work = frame[["ba_code", "hour_local", "month", target]].copy()
@@ -55,14 +26,10 @@ def robust_seasonal_z(frame: pd.DataFrame, target: str = "demand_mwh") -> pd.Ser
 
     median = grouped.transform("median")
     mad = grouped.transform(lambda s: (s - s.median()).abs().median())
-    # 0.6745 rescales MAD to be comparable with a standard deviation under normality.
     scale = (mad / 0.6745).replace(0, np.nan)
     return ((work[target] - median) / scale).abs().fillna(0.0)
 
 
-# ---------------------------------------------------------------------------
-# Detector 2: Isolation Forest
-# ---------------------------------------------------------------------------
 ISO_FEATURES = ["demand_mwh", "ramp_mwh", "ramp_pct", "temperature_2m", "demand_per_degree"]
 
 
@@ -70,8 +37,6 @@ def _isolation_features(frame: pd.DataFrame) -> pd.DataFrame:
     work = frame.copy()
     work["ramp_mwh"] = work.groupby("ba_code")["demand_mwh"].diff()
     work["ramp_pct"] = work["ramp_mwh"] / work.groupby("ba_code")["demand_mwh"].shift(1) * 100
-    # Demand normalised by distance from the comfort balance point: how much load
-    # each degree of heating or cooling demand is buying.
     departure = (work["temperature_2m"] - 18.0).abs().clip(lower=0.5)
     work["demand_per_degree"] = work["demand_mwh"] / departure
     return work[ISO_FEATURES].replace([np.inf, -np.inf], np.nan)
@@ -92,24 +57,13 @@ def isolation_forest_scores(frame: pd.DataFrame, contamination: float = CONTAMIN
         ),
     )
     model.fit(features)
-    # decision_function is high for normal points; negate so high means anomalous.
     raw = -model[-1].decision_function(model[:-1].transform(features))
     return pd.Series(raw, index=frame.index), model
 
 
-# ---------------------------------------------------------------------------
-# Detector 3: daily-profile autoencoder
-# ---------------------------------------------------------------------------
 def _daily_profiles(frame: pd.DataFrame) -> tuple[np.ndarray, pd.DataFrame]:
-    """Reshape into one normalised 24-value vector per (BA, local date).
-
-    Each day is divided by its own **median** so the autoencoder learns load
-    *shape* rather than which BA is largest. The median is used instead of the
-    mean because a single corrupt hour drags a mean toward itself; if that mean
-    lands near zero the division explodes and the reconstruction loss becomes
-    meaningless. Days whose level is not comfortably positive are dropped rather
-    than rescued, since their shape cannot be trusted anyway.
-    """
+    """Turn each day into 24 numbers scaled by that day's median, so the model
+    learns the shape of a day rather than which region is biggest."""
     pivot = (
         frame.pivot_table(index=["ba_code", "date_local"], columns="hour_local",
                           values="demand_mwh", aggfunc="mean")
@@ -130,8 +84,6 @@ def _daily_profiles(frame: pd.DataFrame) -> tuple[np.ndarray, pd.DataFrame]:
         return np.empty((0, 24), dtype=np.float32), pd.DataFrame()
 
     normalised = values / level
-    # A normalised day should sit near 1.0 throughout. Anything beyond this band
-    # is a corrupt reading, not a load shape, and would dominate the loss.
     keep = (normalised > 0.05).all(axis=1) & (normalised < 20.0).all(axis=1)
     dropped = int((~keep).sum())
     if dropped:
@@ -160,7 +112,7 @@ def autoencoder_scores(frame: pd.DataFrame, quick: bool = False) -> pd.DataFrame
 
     model = nn.Sequential(
         nn.Linear(n_hours, 32), nn.ReLU(),
-        nn.Linear(32, 8), nn.ReLU(),          # bottleneck
+        nn.Linear(32, 8), nn.ReLU(),
         nn.Linear(8, 32), nn.ReLU(),
         nn.Linear(32, n_hours),
     )
@@ -194,9 +146,6 @@ def autoencoder_scores(frame: pd.DataFrame, quick: bool = False) -> pd.DataFrame
     return out
 
 
-# ---------------------------------------------------------------------------
-# Orchestration
-# ---------------------------------------------------------------------------
 def classify_anomaly(row: pd.Series) -> str:
     """Human-readable label so an operator knows what they are looking at."""
     if row.get("flag_frozen_reading"):
@@ -230,29 +179,23 @@ def run_anomaly_detection(quick: bool = False, persist: bool = True) -> pd.DataF
 
     frame["period_utc"] = pd.to_datetime(frame["period_utc"], utc=True)
 
-    # Physically impossible readings are already flagged by the warehouse and
-    # reported by the quality suite. Feeding them to statistical detectors would
-    # let them define the very distribution used to judge everything else.
     implausible = frame["flag_implausible_magnitude"].fillna(False).astype(bool)
     if implausible.any():
         logger.info("  excluding %d implausible reading(s) from scoring", int(implausible.sum()))
     scored = frame[frame["demand_mwh"].notna() & ~implausible].copy()
     logger.info("Scoring %s hours for anomalies", f"{len(scored):,}")
 
-    # Detector 1
     logger.info("  detector 1/3: robust seasonal z-score")
     scored["robust_z"] = robust_seasonal_z(scored)
     grouped = scored.groupby(["ba_code", "hour_local", "month"])["demand_mwh"]
     scored["residual_sign"] = np.sign(scored["demand_mwh"] - grouped.transform("median"))
 
-    # Detector 2
     logger.info("  detector 2/3: isolation forest")
     scored["iso_score"], _ = isolation_forest_scores(scored)
     scored["iso_flag"] = scored["iso_score"] > scored["iso_score"].quantile(1 - CONTAMINATION)
     scored["ramp_mwh"] = scored.groupby("ba_code")["demand_mwh"].diff()
     scored["ramp_pct"] = scored["ramp_mwh"] / scored.groupby("ba_code")["demand_mwh"].shift(1) * 100
 
-    # Detector 3
     logger.info("  detector 3/3: daily-profile autoencoder")
     daily = autoencoder_scores(scored, quick=quick)
     if not daily.empty:
@@ -266,7 +209,6 @@ def run_anomaly_detection(quick: bool = False, persist: bool = True) -> pd.DataF
 
     scored["ae_anomalous_day"] = scored["ae_anomalous_day"].astype("boolean").fillna(False).astype(bool)
 
-    # Consensus
     scored["z_flag"] = scored["robust_z"] > Z_THRESHOLD
     def _as_bool(column: str) -> pd.Series:
         return scored[column].astype("boolean").fillna(False).astype(bool)

src/gridpulse/models/baselines.py

@@ -1,22 +1,4 @@
-"""Naive and classical statistical baselines.
-
-Every forecasting claim is meaningless without a floor to compare against. These
-three establish it:
-
-**Seasonal naive (24h)**
-    Tomorrow at 3pm equals today at 3pm. Trivial, and startlingly hard to beat.
-
-**Weekly naive (168h)**
-    Tomorrow at 3pm equals the same weekday last week at 3pm. Usually stronger than
-    the daily variant because it preserves the weekday/weekend regime.
-
-**Holt-Winters**
-    Triple exponential smoothing with a daily seasonal cycle: a real statistical
-    model, no exogenous inputs, representing what a utility analyst could build in
-    a spreadsheet.
-
-If a deep network cannot beat weekly naive, the deep network is not working.
-"""
+"""Simple baselines to beat: seasonal naive, weekly naive and Holt-Winters."""
 
 from __future__ import annotations
 

src/gridpulse/models/deep.py

@@ -1,32 +1,4 @@
-"""The deep learning models, written in PyTorch and built to run on a CPU.
-
-How they are put together
--------------------------
-Both models use the same idea as the Temporal Fusion Transformer, cut down to
-something that trains in minutes on a laptop instead of hours on a GPU:
-
-* An **encoder** reads the last 168 hours of what actually happened: demand,
-  weather, and the time of day and week.
-* A **second branch** reads the next 24 hours of things we already know, which is
-  the weather forecast and the calendar. This is not cheating. A real grid
-  operator making a day-ahead forecast genuinely has tomorrow's weather forecast
-  and knows what day of the week it is. Hiding that would mean solving a harder
-  problem than the real one.
-* A **final layer** combines the two and predicts all 24 hours at once, instead of
-  predicting one hour and feeding it back in. Feeding predictions back in makes
-  small errors pile up.
-
-There are two versions of the encoder: a stacked LSTM, and a small Transformer
-with sinusoidal position encoding. At this amount of data the LSTM usually does
-better. I included the Transformer because it scales better when you have far more
-series, and because I wanted to actually build the attention part rather than just
-read about it.
-
-One thing I had to be careful about is memory. The training windows are never all
-built at once. The dataset keeps one flat float32 array and cuts each window out
-of it only when it is asked for, so memory stays in the tens of megabytes no
-matter how much history there is.
-"""
+"""PyTorch LSTM and Transformer forecasters, sized to train on a laptop CPU."""
 
 from __future__ import annotations
 
@@ -43,7 +15,6 @@ from gridpulse.config import FORECAST_HORIZON, LOOKBACK_HOURS, PATHS
 
 logger = logging.getLogger(__name__)
 
-# Channels observed in the past and fed to the encoder.
 PAST_CHANNELS = [
     "demand_mwh",
     "temperature_2m", "apparent_temperature", "relative_humidity_2m",
@@ -53,7 +24,6 @@ PAST_CHANNELS = [
     "is_business_day", "is_holiday",
 ]
 
-# Channels known in advance for the forecast window.
 FUTURE_CHANNELS = [
     "temperature_2m", "apparent_temperature", "relative_humidity_2m",
     "cloud_cover", "wind_speed_10m",
@@ -64,33 +34,14 @@ FUTURE_CHANNELS = [
 
 TARGET = "demand_mwh"
 
-# Sliding windows at hourly resolution overlap by 167 of 168 input hours, so
-# adjacent samples are almost perfectly redundant. Striding keeps the sample
-# diverse while cutting epoch time by the stride factor. Six hours is a natural
-# choice: it still covers every phase of the daily cycle within a single day.
 TRAIN_STRIDE = 12
 QUICK_TRAIN_STRIDE = 24
 
-# Recurrent layers are inherently sequential: timestep t cannot be computed until
-# t-1 finishes, so cost scales linearly with sequence length and cannot be
-# parallelised away. A 168-step encoder is therefore expensive on a CPU.
-#
-# The lookback window is subsampled every ENCODER_STRIDE hours, giving 56 steps
-# instead of 168. This is not a shortcut: hourly demand is heavily autocorrelated,
-# so consecutive hours carry little independent information, and the retained
-# points still span the full week and every phase of the daily cycle. The recent
-# past is preserved exactly where it matters most through the lag and rolling
-# features already supplied to the gradient-boosted model.
 ENCODER_STRIDE = 3
 
-# Each test window predicts 24 consecutive hours, so a stride of 6 still yields
-# four independent predictions for every hour, which are averaged.
 TEST_STRIDE = 6
 
 
-# ---------------------------------------------------------------------------
-# Dataset
-# ---------------------------------------------------------------------------
 def _torch():
     try:
         import torch
@@ -133,7 +84,6 @@ class WindowDataset:
         split = start + self.lookback
         end = split + self.horizon
         return (
-            # Subsampled: 168 hourly steps become 56 three-hourly steps.
             torch.from_numpy(self.past[start:split:ENCODER_STRIDE]),
             torch.from_numpy(self.future[split:end]),
             torch.from_numpy(self.target[split:end]),
@@ -155,9 +105,6 @@ class ConcatDataset:
         return self.datasets[which][index - int(self.offsets[which])]
 
 
-# ---------------------------------------------------------------------------
-# Models
-# ---------------------------------------------------------------------------
 def build_lstm(n_past: int, n_future: int, hidden: int = 64, layers: int = 1, dropout: float = 0.15):
     torch = _torch()
     nn = torch.nn
@@ -179,9 +126,9 @@ def build_lstm(n_past: int, n_future: int, hidden: int = 64, layers: int = 1, dr
 
         def forward(self, past, future):
             _, (hidden_state, _) = self.encoder(past)
-            context = hidden_state[-1]                                   # (B, hidden)
-            known = self.future_proj(future.flatten(start_dim=1))        # (B, hidden)
-            return self.head(torch.cat([context, known], dim=1))         # (B, horizon)
+            context = hidden_state[-1]
+            known = self.future_proj(future.flatten(start_dim=1))
+            return self.head(torch.cat([context, known], dim=1))
 
     return LSTMForecaster()
 
@@ -225,16 +172,13 @@ def build_transformer(
 
         def forward(self, past, future):
             encoded = self.encoder(self.pos(self.input_proj(past)))
-            context = encoded.mean(dim=1)                                # attention-pooled summary
+            context = encoded.mean(dim=1)
             known = self.future_proj(future.flatten(start_dim=1))
             return self.head(torch.cat([context, known], dim=1))
 
     return TransformerForecaster()
 
 
-# ---------------------------------------------------------------------------
-# Scaling
-# ---------------------------------------------------------------------------
 @dataclass
 class Scaler:
     """Per-channel standardisation. Statistics come from training data only."""
@@ -262,22 +206,13 @@ class Scaler:
 
 @dataclass
 class TargetScaler:
-    """Per-BA target scaling.
-
-    BA demand spans two orders of magnitude (ISNE peaks near 25 GW, PJM near
-    150 GW). Without per-BA normalisation the loss is dominated entirely by the
-    largest system and the small ones never learn.
-    """
+    """Scales demand per region, so the biggest regions do not dominate training."""
 
     stats: dict[str, tuple[float, float]]
 
     @classmethod
     def fit(cls, frame: pd.DataFrame) -> TargetScaler:
-        """Robust per-BA centre and scale (median / IQR).
-
-        See ``gbm.BATargetScaler.fit`` for why mean and standard deviation are
-        unusable here: corrupt readings in the raw feed inflate them without limit.
-        """
+        """Median and IQR per region, so one bad reading cannot skew the scaling."""
         grouped = frame.groupby("ba_code")[TARGET]
         centre = grouped.median()
         spread = (grouped.quantile(0.75) - grouped.quantile(0.25)) / 1.349
@@ -305,18 +240,15 @@ class TargetScaler:
         return cls({k: (v[0], v[1]) for k, v in payload["stats"].items()})
 
 
-# ---------------------------------------------------------------------------
-# Windowing across the full series
-# ---------------------------------------------------------------------------
 @dataclass
 class SeriesBundle:
     """Everything needed to build windows for a single balancing authority."""
 
     ba_code: str
-    past: np.ndarray          # (T, n_past)  scaled
-    future: np.ndarray        # (T, n_future) scaled
-    target_scaled: np.ndarray  # (T,)         scaled
-    target_raw: np.ndarray    # (T,)         MWh
+    past: np.ndarray
+    future: np.ndarray
+    target_scaled: np.ndarray
+    target_raw: np.ndarray
     timestamps: pd.DatetimeIndex
 
 
@@ -346,12 +278,10 @@ def split_windows(
     train_stride: int = TRAIN_STRIDE,
     test_stride: int = TEST_STRIDE,
 ) -> tuple[ConcatDataset, ConcatDataset, list[tuple[SeriesBundle, np.ndarray]]]:
-    """Assign every window to a split by the timestamp of its first forecast hour.
+    """Put each training window into train, validation or test by its date.
 
-    Windows are cut over the *full* contiguous series and only then partitioned, so
-    no usable window is lost at a split boundary. A window is assigned to test only
-    when its entire forecast horizon lies in the test period, which makes the
-    evaluation strictly out-of-sample.
+    A window only counts as test if all 24 forecast hours fall in the test period,
+    so nothing the model saw during training leaks into the score.
     """
     train_sets, valid_sets, test_specs = [], [], []
 
@@ -397,9 +327,6 @@ class _SubsetWindows(WindowDataset):
         return super().__getitem__(int(self.indices[index]))
 
 
-# ---------------------------------------------------------------------------
-# Training
-# ---------------------------------------------------------------------------
 @dataclass
 class TrainedDeepModel:
     architecture: str
@@ -513,8 +440,6 @@ def train_deep(
 
     optimiser = torch.optim.AdamW(model.parameters(), lr=2e-3, weight_decay=1e-4)
     scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimiser, factor=0.5, patience=2)
-    # Huber is deliberately chosen over MSE: load series contain genuine spikes
-    # (heatwaves, storms) and MSE would let a handful of them dominate the gradient.
     criterion = torch.nn.HuberLoss(delta=1.0)
 
     max_epochs = 4 if quick else 15
@@ -573,11 +498,7 @@ def train_deep(
 
 
 def _predict_test(model, test_specs, target_scaler: TargetScaler, batch_size: int = 256) -> pd.DataFrame:
-    """Score every test window and flatten to one row per (BA, timestamp).
-
-    Overlapping windows produce several predictions for the same hour; they are
-    averaged, which is a cheap ensembling effect and smooths window-edge artefacts.
-    """
+    """Predict every test window, averaging where windows overlap the same hour."""
     torch = _torch()
     model.eval()
     rows = []

src/gridpulse/models/gbm.py

@@ -1,27 +1,4 @@
-"""LightGBM demand forecasting, with P10/P50/P90 prediction bands.
-
-I train one model across all 12 regions, with the region code as a categorical
-feature, instead of 12 separate models. Reasons:
-
-* The regions behave similarly. How demand responds to temperature in Atlanta
-  really does tell you something about the same curve in Charlotte, so training
-  together helps the smaller regions.
-* One model file is one thing to deploy, version and keep an eye on. Twelve
-  models are twelve of everything.
-* Adding a thirteenth region is then just more data, not more infrastructure.
-
-This only works if the demand is scaled first. The regions differ by a factor of
-about a hundred (ISNE peaks around 25 GW, PJM around 150 GW), so if you train on
-raw megawatthours the loss is completely dominated by the biggest regions. The
-model spends all its effort on PJM and never learns the small ones. Scaling each
-region separately puts them on a comparable footing, and predictions get converted
-back to MW before anything is scored. Without this step the model gives up after a
-handful of trees, which is exactly what happened before I added it.
-
-As well as the single number forecast, three more models predict the 10th, 50th
-and 90th percentiles. Utilities do not plan against a single number, they plan
-against a realistic worst case, so the range is the more useful output.
-"""
+"""One LightGBM model across all 12 regions, plus P10/P50/P90 prediction bands."""
 
 from __future__ import annotations
 
@@ -43,12 +20,7 @@ CATEGORICAL = ["ba_code"]
 
 @dataclass
 class BATargetScaler:
-    """Scales demand separately for each region, so they can share one model.
-
-    The numbers are worked out from the training rows only, never the test rows.
-    ``inverse`` converts predictions back into megawatthours, so every metric
-    downstream is reported in real units rather than scaled ones.
-    """
+    """Scales demand separately for each region so they can share one model."""
 
     stats: dict[str, tuple[float, float]]
     global_mean: float
@@ -56,20 +28,9 @@ class BATargetScaler:
 
     @classmethod
     def fit(cls, frame: pd.DataFrame) -> BATargetScaler:
-        """Work out the middle and the spread of demand for each region.
-
-        I use the median and the IQR here instead of the mean and standard
-        deviation. One impossible reading can drag a mean anywhere it likes, and
-        the real EIA data does occasionally contain values that are wildly too
-        big. A single row like that pushed PJM's standard deviation up to 10.7
-        million MW when the real range is about 70,000 to 165,000 MW, which
-        wrecked the scaling and every prediction that depended on it. The median
-        does not move no matter how extreme one value is.
-        """
+        """Median and IQR per region, so one bad reading cannot skew the scaling."""
         grouped = frame.groupby("ba_code")[TARGET]
         centre = grouped.median()
-        # Dividing the IQR by 1.349 puts it on roughly the same scale as a
-        # standard deviation would be, so the scaled values stay familiar.
         spread = (grouped.quantile(0.75) - grouped.quantile(0.25)) / 1.349
 
         stats = {
@@ -154,7 +115,6 @@ class TrainedGBM:
     best_iteration: int
     target_scaler: BATargetScaler
 
-    # -- persistence ------------------------------------------------------
     def save(self, directory: Path | None = None) -> Path:
         target = Path(directory) if directory else PATHS.artifacts / "gbm"
         target.mkdir(parents=True, exist_ok=True)
@@ -196,7 +156,6 @@ class TrainedGBM:
             target_scaler=BATargetScaler.from_dict(meta["target_scaler"]),
         )
 
-    # -- inference --------------------------------------------------------
     def predict(self, frame: pd.DataFrame) -> pd.DataFrame:
         """Point and quantile predictions, returned in megawatthours.
 
@@ -212,7 +171,6 @@ class TrainedGBM:
             out[f"pred_gbm_p{int(q * 100)}"] = self.target_scaler.inverse(
                 model.predict(matrix), ba_codes
             )
-        # Quantile models are fit independently and can cross; enforce monotonicity.
         quantile_columns = [c for c in out.columns if c.startswith("pred_gbm_p")]
         out[quantile_columns] = np.sort(out[quantile_columns].to_numpy(), axis=1)
         return out
@@ -257,8 +215,6 @@ def train_gbm(
     ba_categories = sorted(pd.concat([train["ba_code"], valid["ba_code"]]).unique().tolist())
     features = list(features) if features else list(FEATURE_COLUMNS)
 
-    # Fitted on training rows only, so no validation information leaks into the
-    # normalisation constants.
     target_scaler = BATargetScaler.fit(train)
 
     x_train = prepare_matrix(train, ba_categories, features)
@@ -286,9 +242,6 @@ def train_gbm(
     )
     logger.info("  point model stopped at iteration %d", point.best_iteration)
 
-    # Quantile models are three additional fits. They are capped well below the
-    # point model's round count: interval edges need far less resolution than the
-    # central estimate, and the extra rounds cost real minutes on a CPU.
     quantile_rounds = int(min(max(200, point.best_iteration), 250 if quick else 700))
     quantile_models: dict[float, object] = {}
     for q in quantiles or ():

src/gridpulse/models/inference.py

@@ -1,19 +1,7 @@
-"""Live inference: produce a genuine forward-looking 24-hour forecast.
-
-Backtesting on stored history is easy. Forecasting *tomorrow* is the part that
-usually breaks, because it needs three things stitched together at request time:
-
-1. **Recent observed demand** for the autoregressive lags, from the warehouse and
-   topped up from the live EIA API when the stored copy is stale.
-2. **Future weather**, fetched live from Open-Meteo. This is legitimate: a system
-   operator producing a day-ahead forecast genuinely holds tomorrow's numerical
-   weather prediction.
-3. **A trained model**, loaded from the committed artifacts so no retraining
-   happens on the request path.
-
-When the network is unavailable the module degrades to *replay* mode: it forecasts
-the most recent 24 hours that already exist in the warehouse, so the public site
-still demonstrates the model instead of showing an error page.
+"""Makes a real forward-looking 24-hour forecast for the live app.
+
+Falls back to replaying the most recent stored day when the network is down, so
+the public site still shows something instead of an error.
 """
 
 from __future__ import annotations
@@ -30,15 +18,15 @@ from gridpulse.features.build import build_features
 
 logger = logging.getLogger(__name__)
 
-HISTORY_HOURS = 400  # comfortably covers the 336-hour deepest lag plus slack
+HISTORY_HOURS = 400
 
 
 @dataclass
 class Forecast:
     ba_code: str
     generated_at_utc: datetime
-    mode: str                      # "live" or "replay"
-    frame: pd.DataFrame            # period_utc, forecast_mwh, p10, p90, [actual_mwh]
+    mode: str
+    frame: pd.DataFrame
     model: str = "gbm"
     notes: list[str] = None
 
@@ -135,9 +123,6 @@ def forecast(ba_code: str, horizon: int = FORECAST_HORIZON, allow_network: bool
     history["period_utc"] = pd.to_datetime(history["period_utc"], utc=True)
     last_observed = history["period_utc"].max()
 
-    # ------------------------------------------------------------------
-    # Assemble the future rows
-    # ------------------------------------------------------------------
     mode = "replay"
     future = pd.DataFrame()
 
@@ -168,22 +153,14 @@ def forecast(ba_code: str, horizon: int = FORECAST_HORIZON, allow_network: bool
         combined = pd.concat([history, future], ignore_index=True)
         target_periods = future["period_utc"]
     else:
-        # Replay: hide the final `horizon` actuals from the feature builder and
-        # predict them, so the chart still shows prediction against truth.
         combined = history.copy()
         target_periods = combined["period_utc"].tail(horizon)
         combined.loc[combined["period_utc"].isin(target_periods), "demand_mwh"] = np.nan
 
     combined = combined.sort_values("period_utc").reset_index(drop=True)
 
-    # Calendar attributes are derived here for every row, historical and future
-    # alike, rather than read from storage. Storage and derivation would otherwise
-    # be two sources of truth for the same values.
     combined = _calendar_columns(combined, BALANCING_AUTHORITIES[ba_code].timezone)
 
-    # ------------------------------------------------------------------
-    # Features and prediction
-    # ------------------------------------------------------------------
     featured = build_features(frame=combined, dropna_target=False)
     horizon_rows = featured[featured["period_utc"].isin(target_periods)].copy()
     if horizon_rows.empty:

src/gridpulse/models/metrics.py

@@ -1,15 +1,4 @@
-"""The metrics used to score the forecasts, chosen to match how utilities score them.
-
-MAPE is what the power industry normally uses, so it comes first here. Someone who
-runs a grid will say "we run about 2 percent MAPE" and everyone knows what that
-means. But I report MAE and RMSE next to it, because MAPE on its own hides the fact
-that some mistakes cost much more than others. RMSE punishes the big misses, which
-are the ones that force an expensive backup plant to start up, while MAPE treats
-being 500 MW off at 3am the same as being 500 MW off at 5pm in a heatwave.
-
-``skill_vs_benchmark`` turns the accuracy into a percentage improvement over EIA's
-own published forecast, which is the comparison that actually matters here.
-"""
+"""Forecast accuracy metrics: MAPE, sMAPE, MAE, RMSE, R2, pinball loss and skill."""
 
 from __future__ import annotations
 
@@ -61,12 +50,7 @@ def r2(y_true, y_pred) -> float:
 
 
 def peak_hour_mape(frame: pd.DataFrame, actual: str, predicted: str) -> float:
-    """MAPE calculated only on the busiest hour of each day.
-
-    The peak hour is what decides how much generation gets bought, and it is where
-    being wrong costs the most money, so I score it separately from the average
-    across all hours.
-    """
+    """MAPE calculated only on the busiest hour of each day."""
     if frame.empty:
         return float("nan")
     peaks = frame.loc[frame.groupby(frame["period_utc"].dt.date)[actual].idxmax()]

src/gridpulse/models/pipeline.py

@@ -1,20 +1,7 @@
-"""Runs the training: builds the features, fits every model, scores them the same way.
-
-Comparing them fairly is the whole point. Six forecasters are tested on exactly the
-same set of held-out hours, and the one they are compared against is not a baseline
-I made up. It is the day-ahead forecast the EIA published and grid operators
-actually used:
-
-1. Seasonal naive, meaning "same hour yesterday"
-2. Weekly naive, meaning "same hour last week"
-3. **EIA's official day-ahead forecast**  <- the one to beat
-4. LightGBM trained on all regions, with P10/P50/P90 bands
-5. LSTM, which also gets tomorrow's weather and calendar
-6. Transformer, same inputs as the LSTM
-
-Every model is scored on the same rows, over the same 24 hours ahead, with the same
-metrics. The results go into ``model_scores`` and ``model_predictions``, and each
-run is logged to MLflow.
+"""Builds the features, trains every model, and scores them all on the same rows.
+
+The benchmark is EIA's own published day-ahead forecast, not a baseline invented
+here. Results go to ``model_scores`` and ``model_predictions``, and to MLflow.
 """
 
 from __future__ import annotations
@@ -53,9 +40,6 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
         mlflow.set_tracking_uri("file:" + str((PATHS.artifacts.parent / "mlruns").as_posix()))
         mlflow.set_experiment("gridpulse-day-ahead-load")
 
-    # ------------------------------------------------------------------
-    # Features and the single shared time split
-    # ------------------------------------------------------------------
     logger.info("Building feature matrix")
     frame = build_features(ba_codes=bas)
     if frame.empty:
@@ -77,9 +61,6 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
 
     predictions = test[["period_utc", "ba_code", "demand_mwh", "demand_forecast_mwh"]].copy()
 
-    # ------------------------------------------------------------------
-    # 1-3. Baselines and the EIA benchmark
-    # ------------------------------------------------------------------
     logger.info("Scoring baselines")
     with_baselines = baselines.build_all_baselines(frame)
     baseline_test = with_baselines[with_baselines["period_utc"] >= test_start]
@@ -87,9 +68,6 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
         if column in baseline_test.columns:
             predictions[column] = baseline_test[column].to_numpy()
 
-    # ------------------------------------------------------------------
-    # 4. LightGBM
-    # ------------------------------------------------------------------
     logger.info("Training LightGBM")
     from gridpulse.models.gbm import train_gbm
 
@@ -99,19 +77,6 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
         predictions[column] = gbm_predictions[column].to_numpy()
     gbm.save()
 
-    # ------------------------------------------------------------------
-    # 4b. Hybrid version: the same model, but it also gets to read EIA's
-    # published day-ahead forecast as one of its inputs.
-    #
-    # This is not cheating. EIA publishes that forecast the day before, so it
-    # really is available at prediction time, and no real utility ignores a
-    # forecast they already have. This version learns to correct the mistakes EIA
-    # consistently makes, instead of working the whole thing out from scratch,
-    # which is closer to how a real forecasting team works anyway.
-    #
-    # I report both. The plain model answers "can I beat them starting from
-    # nothing", and the hybrid answers "can I improve on what they publish".
-    # ------------------------------------------------------------------
     if train["demand_forecast_mwh"].notna().mean() > 0.9:
         logger.info("Training LightGBM hybrid (EIA forecast as an input feature)")
         from gridpulse.features.build import FEATURE_COLUMNS
@@ -129,9 +94,6 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
     )
     logger.info("Top features:\n%s", importance.head(12).to_string(index=False))
 
-    # ------------------------------------------------------------------
-    # 5-6. Deep models
-    # ------------------------------------------------------------------
     for architecture in ("lstm", "transformer"):
         try:
             logger.info("Training deep model: %s", architecture)
@@ -154,19 +116,12 @@ def train_all(bas: list[str] | None = None, quick: bool = False) -> pd.DataFrame
         except Exception as exc:  # noqa: BLE001
             logger.error("Deep model %s failed: %s", architecture, exc, exc_info=True)
 
-    # ------------------------------------------------------------------
-    # Ensemble: simple average of the two strongest model families
-    # ------------------------------------------------------------------
     ensemble_parts = [c for c in ("pred_gbm", "pred_lstm") if c in predictions.columns]
     if len(ensemble_parts) > 1:
         predictions["pred_ensemble"] = predictions[ensemble_parts].mean(axis=1)
 
-    # ------------------------------------------------------------------
-    # Scoring
-    # ------------------------------------------------------------------
     leaderboard = _score(predictions)
 
-    # Interval calibration, reported only if the quantile models produced output.
     if {"pred_gbm_p10", "pred_gbm_p90"} <= set(predictions.columns):
         interval_coverage = metrics.coverage(
             predictions["demand_mwh"], predictions["pred_gbm_p10"], predictions["pred_gbm_p90"]

src/gridpulse/quality/checks.py

@@ -1,17 +1,5 @@
-"""The data quality checks.
-
-Electricity meter data goes wrong in ways that general purpose testing tools do not
-look for. A meter reporting the exact same value for six hours is not steady, it is
-stuck. An hour that disappears every March is not missing data, it is the clocks
-going forward. A negative demand reading does not mean the grid was quiet, it means
-someone got a plus and minus the wrong way round further upstream. Every check
-below is written around one of those specific problems.
-
-Each check is scored against one of six categories: completeness, validity,
-consistency, timeliness, duplicates and accuracy. The results get saved into
-``dq_results`` and ``dq_scorecard`` rather than just printed, so I can look back at
-how data quality changed over time instead of only seeing today's answer.
-"""
+"""Sixteen data quality checks, scored across six categories and saved to the
+warehouse so quality can be tracked over time rather than only printed."""
 
 from __future__ import annotations
 
@@ -28,9 +16,9 @@ logger = logging.getLogger(__name__)
 
 
 class Severity(str, Enum):
-    CRITICAL = "critical"  # fails the pipeline
-    WARNING = "warning"    # logged and tracked, does not block
-    INFO = "info"          # observability only
+    CRITICAL = "critical"
+    WARNING = "warning"
+    INFO = "info"
 
 
 class Dimension(str, Enum):
@@ -44,18 +32,15 @@ class Dimension(str, Enum):
 
 @dataclass(frozen=True)
 class Check:
-    """A single quality assertion.
-
-    ``sql`` must return exactly one row with columns ``failed`` and ``total``.
-    The check passes when ``failed / total <= threshold``.
-    """
+    """One check. Its SQL returns ``failed`` and ``total``, and it passes when
+    ``failed / total`` is within the threshold."""
 
     name: str
     dimension: Dimension
     severity: Severity
     description: str
     sql: str
-    threshold: float = 0.0  # tolerated failure fraction
+    threshold: float = 0.0
 
 
 @dataclass
@@ -117,9 +102,6 @@ class QualityReport:
         )
 
 
-# ---------------------------------------------------------------------------
-# The suite
-# ---------------------------------------------------------------------------
 CHECKS: list[Check] = [
     Check(
         name="demand_not_null",
@@ -127,7 +109,7 @@ CHECKS: list[Check] = [
         severity=Severity.WARNING,
         description="Actual demand is reported for every hour on the spine.",
         sql="SELECT count(*) FILTER (WHERE demand_mwh IS NULL) AS failed, count(*) AS total FROM fact_demand_hourly",
-        threshold=0.02,  # BAs genuinely miss the occasional hour
+        threshold=0.02,
     ),
     Check(
         name="demand_positive",
@@ -316,20 +298,7 @@ CHECKS: list[Check] = [
 
 
 def run_quality_suite(persist: bool = True, database=None) -> QualityReport:
-    """Execute every check and optionally persist the results.
-
-    Parameters
-    ----------
-    persist
-        Write results to ``dq_results`` and rebuild ``dq_scorecard``.
-    database
-        Override the warehouse path. Used by the test suite.
-
-    Returns
-    -------
-    QualityReport
-        ``report.passed`` is False when any CRITICAL check failed.
-    """
+    """Run every check. ``report.passed`` is False if any critical one failed."""
     report = QualityReport()
 
     with connect(database, read_only=not persist) as con:

src/gridpulse/warehouse/build.py

@@ -1,26 +1,7 @@
-"""Turning raw downloads into the warehouse: bronze, then silver, then gold.
+"""Builds the warehouse in DuckDB: raw bronze, cleaned silver, star schema gold.
 
-Each of the three layers has one job:
-
-**Bronze**
-    The raw data exactly as it was downloaded, one row per region per hour per
-    measurement. I never edit it, only add to it, so if something goes wrong later
-    I can always go back and reproduce it.
-
-**Silver**
-    Cleaned up and reshaped. The measurements become columns, the weather gets
-    joined on, a full list of every hour makes any missing hours obvious, local
-    time is worked out for each region, and anything suspicious is *flagged rather
-    than deleted*. Deleting a bad reading also deletes the proof that a meter was
-    broken, and that proof is the useful part.
-
-**Gold**
-    A star schema, which is what the dashboard and the models read from. It has
-    lookup tables around `fact_demand_hourly`, plus a separate table that scores
-    EIA's own published forecast against what actually happened.
-
-All the heavy work is done in SQL over whole tables at once, rather than looping
-through rows in pandas, so memory stays flat no matter how much history there is.
+Bad readings are flagged rather than deleted, because deleting them also deletes
+the evidence that a meter was broken.
 """
 
 from __future__ import annotations
@@ -34,23 +15,10 @@ from gridpulse.warehouse.duck import connect, row_count, table_exists
 
 logger = logging.getLogger(__name__)
 
-# Physically implausible demand. Real BA demand never legitimately hits zero;
-# a zero or negative reading is a telemetry failure, not a quiet grid.
 MIN_PLAUSIBLE_MWH = 1.0
-# Hour-on-hour swings beyond this are almost always bad data, not real load.
 MAX_HOURLY_RAMP_PCT = 40.0
 
-# I find spikes by comparing each point against a rolling median centred on it,
-# rather than against the point before and after. Comparing to neighbours broke in
-# two ways when I tried it. It needs a threshold low enough to catch a spike that
-# is only extreme on one side, but high enough not to flag genuine fast changes.
-# And it cannot judge the very first or very last row at all, which is exactly
-# where the newest and least reliable data sits.
-#
-# A rolling median has neither problem. Total demand moves smoothly over five
-# hours, so a normal day never strays far from its local median, while a single
-# bad reading stands out no matter which side it falls on.
-SPIKE_WINDOW_HOURS = 2      # rows either side, so a 5 hour window centred on each point
+SPIKE_WINDOW_HOURS = 2
 SPIKE_DEVIATION_PCT = 20.0
 
 
@@ -100,7 +68,7 @@ def _dim_date_frame(start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
     frame["quarter"] = days.quarter
     frame["month"] = days.month
     frame["day_of_month"] = days.day
-    frame["day_of_week"] = days.dayofweek           # Monday = 0
+    frame["day_of_week"] = days.dayofweek
     frame["day_of_year"] = days.dayofyear
     frame["week_of_year"] = days.isocalendar().week.astype(int)
     frame["is_weekend"] = days.dayofweek >= 5
@@ -151,10 +119,6 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
 
     with connect() as con:
         if rebuild:
-            # Only the tables this function owns are dropped. Model scores,
-            # predictions and anomaly results belong to other commands; wiping
-            # them here would silently discard an hour of training because
-            # someone rebuilt the data layer.
             logger.info("Rebuild requested: dropping data-layer tables")
             for table in (
                 "fact_forecast_accuracy", "fact_demand_hourly",
@@ -173,17 +137,11 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
                     ", ".join(stale),
                 )
 
-        # ------------------------------------------------------------------
-        # Dimensions
-        # ------------------------------------------------------------------
         logger.info("Building dim_ba")
         con.register("_dim_ba", _dim_ba_frame())
         con.execute("CREATE OR REPLACE TABLE dim_ba AS SELECT * FROM _dim_ba")
         con.unregister("_dim_ba")
 
-        # ------------------------------------------------------------------
-        # Silver: pivot measures, attach weather, derive local civil time
-        # ------------------------------------------------------------------
         logger.info("Building silver_grid_hourly")
         con.execute(f"""
         CREATE OR REPLACE TEMP TABLE _eia_wide AS
@@ -207,11 +165,6 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
             weather_join = ""
             weather_select = weather_nulls
 
-        # A dense hourly spine per BA. Anti-joining against it is the only reliable
-        # way to distinguish "reported zero" from "never reported at all".
-        # Built in pandas rather than SQL: generate_series over TIMESTAMPTZ depends
-        # on the ICU extension and has shifted across DuckDB releases, whereas
-        # date_range is deterministic on every platform.
         bounds = con.execute(
             "SELECT ba_code, min(period_utc) AS lo, max(period_utc) AS hi "
             "FROM _eia_wide GROUP BY ba_code"
@@ -266,9 +219,6 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
         ORDER BY s.ba_code, s.period_utc
         """)
 
-        # ------------------------------------------------------------------
-        # Gold: calendar dimension sized to the observed data
-        # ------------------------------------------------------------------
         span = con.execute(
             "SELECT min(date_local), max(date_local) FROM silver_grid_hourly"
         ).fetchone()
@@ -283,12 +233,7 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
         )
         con.unregister("_dim_date")
 
-        # ------------------------------------------------------------------
-        # Gold: central fact
-        # ------------------------------------------------------------------
         logger.info("Building fact_demand_hourly")
-        # Robust per-BA bounds. Computed from the median so that the outliers being
-        # detected cannot influence the threshold that detects them.
         con.execute("""
         CREATE OR REPLACE TEMP TABLE _ba_bounds AS
         SELECT ba_code,
@@ -300,8 +245,6 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
         GROUP BY ba_code
         """)
 
-        # Local level for spike detection, computed once so the window function is
-        # not repeated across several expressions.
         con.execute("""
         CREATE OR REPLACE TEMP TABLE _neighbours AS
         SELECT
@@ -367,10 +310,6 @@ def build_warehouse(rebuild: bool = False) -> dict[str, int]:
         ORDER BY s.ba_code, s.period_utc
         """)
 
-        # ------------------------------------------------------------------
-        # Gold: the benchmark table. This is the one that holds EIA's own forecast
-        # next to what actually happened, so anyone can check my headline claim.
-        # ------------------------------------------------------------------
         logger.info("Building fact_forecast_accuracy")
         con.execute("""
         CREATE OR REPLACE TABLE fact_forecast_accuracy AS

src/gridpulse/warehouse/duck.py

@@ -1,10 +1,4 @@
-"""DuckDB connection management.
-
-DuckDB is the warehouse engine because it gives columnar OLAP performance over
-hundreds of millions of rows inside a single embedded file, with no server to run.
-On an 8GB laptop that is the difference between a project that runs and a project
-that swaps. The same SQL ports to Snowflake, BigQuery or Azure Synapse unchanged.
-"""
+"""Opening and closing DuckDB connections, plus a few small query helpers."""
 
 from __future__ import annotations
 
@@ -20,8 +14,6 @@ from gridpulse.config import PATHS
 
 logger = logging.getLogger(__name__)
 
-# Tuned for a constrained laptop: cap DuckDB well below total system RAM so the OS,
-# the Python process and any concurrently running training job all still fit.
 DEFAULT_MEMORY_LIMIT = "2GB"
 DEFAULT_THREADS = 4
 
@@ -41,7 +33,7 @@ def connect(
     try:
         con.execute(f"SET memory_limit='{memory_limit}'")
         con.execute(f"SET threads={threads}")
-        con.execute("SET preserve_insertion_order=false")  # lower peak memory on big sorts
+        con.execute("SET preserve_insertion_order=false")
         yield con
     finally:
         con.close()

src/gridpulse/warehouse/export.py

@@ -1,16 +1,7 @@
-"""Build the slim deployment artifact the public app ships with.
+"""Writes the small DuckDB file the public app ships with, about 13 MB.
 
-The development warehouse holds years of hourly history across every table. Git and
-free-tier hosting do not want that. This module distils it into a compact DuckDB
-file containing only what the public site actually reads:
-
-* a rolling window of recent hourly demand, weather and forecasts,
-* pre-computed model predictions over the evaluation window,
-* the model leaderboard and data quality scorecard,
-* flagged anomalies.
-
-The app therefore starts instantly with no retraining and no warehouse dependency,
-while still calling the live EIA API for anything newer than the last export.
+The full warehouse is 129 MB, which is too big for Git and for free hosting, so
+this keeps only a rolling window of what the website actually reads.
 """
 
 from __future__ import annotations
@@ -26,7 +17,7 @@ from gridpulse.warehouse.duck import connect, row_count
 logger = logging.getLogger(__name__)
 
 APP_DB_NAME = "gridpulse_app.duckdb"
-EXPORT_WINDOW_DAYS = 400  # a little over a year, so the app can show YoY comparisons
+EXPORT_WINDOW_DAYS = 400
 
 
 def export_for_app(window_days: int = EXPORT_WINDOW_DAYS, destination: Path | None = None) -> Path:
@@ -48,16 +39,11 @@ def export_for_app(window_days: int = EXPORT_WINDOW_DAYS, destination: Path | No
     with connect(target) as con:
         con.execute(f"ATTACH '{source.as_posix()}' AS wh (READ_ONLY)")
 
-        # Dimensions are small; copy wholesale.
         for table in ("dim_ba", "dim_date"):
             if _exists_in(con, "wh", table):
                 con.execute(f"CREATE TABLE {table} AS SELECT * FROM wh.{table}")
                 manifest[table] = row_count(con, table)
 
-        # The main fact, trimmed to the rolling window. The column list is derived
-        # from WEATHER_VARIABLES rather than hand-written: a hand-written list
-        # silently drops columns the feature builder needs, and the failure only
-        # surfaces at inference time on the deployed site.
         if _exists_in(con, "wh", "fact_demand_hourly"):
             columns = ", ".join([
                 "period_utc", "ba_code", "date_local", "hour_local",
@@ -122,7 +108,6 @@ def _exists_in(con, schema: str, table: str) -> bool:
     ).fetchone()
     if found and found[0]:
         return True
-    # DuckDB reports attached databases via the catalog column in some versions.
     found = con.execute(
         "SELECT count(*) FROM information_schema.tables WHERE table_catalog = ? AND table_name = ?",
         [schema, table],

tests/conftest.py

@@ -45,7 +45,6 @@ def synthetic_grid() -> pd.DataFrame:
             "period_utc": periods,
             "ba_code": ba,
             "demand_mwh": demand.round(1),
-            # EIA's forecast: the truth plus a realistic ~2 percent error.
             "demand_forecast_mwh": (demand * (1 + rng.normal(0, 0.021, len(periods)))).round(1),
             "net_generation_mwh": (demand * 1.03).round(1),
             "total_interchange_mwh": rng.normal(0, scale * 0.02, len(periods)).round(1),

tests/test_features.py

@@ -93,7 +93,6 @@ def test_rolling_features_do_not_leak_the_present(raw_frame):
 
 def test_degree_days_split_the_temperature_response(raw_frame):
     featured = build_features(frame=raw_frame)
-    # The two limbs are mutually exclusive: never both positive at once.
     both_positive = (featured["heating_degrees"] > 0) & (featured["cooling_degrees"] > 0)
     assert not both_positive.any()
     assert (featured["heating_degrees"] >= 0).all()

tests/test_metrics.py

@@ -17,15 +17,14 @@ def test_perfect_forecast_scores_zero_error():
 
 
 def test_mape_is_computed_correctly():
-    # A uniform 10 percent over-forecast must yield exactly 10 percent MAPE.
     truth = np.array([100.0, 200.0, 400.0])
     assert metrics.mape(truth, truth * 1.1) == pytest.approx(10.0)
 
 
 def test_rmse_penalises_large_errors_more_than_mae():
     truth = np.array([100.0, 100.0, 100.0, 100.0])
-    concentrated = np.array([100.0, 100.0, 100.0, 140.0])  # one big miss
-    spread = np.array([110.0, 110.0, 110.0, 110.0])        # four small misses
+    concentrated = np.array([100.0, 100.0, 100.0, 140.0])
+    spread = np.array([110.0, 110.0, 110.0, 110.0])
     assert metrics.mae(truth, concentrated) == pytest.approx(metrics.mae(truth, spread))
     assert metrics.rmse(truth, concentrated) > metrics.rmse(truth, spread)
 

tests/test_quality.py

@@ -58,7 +58,6 @@ class TestSuiteExecution:
             for r in report.results
             if r.check.severity is Severity.CRITICAL
             and not r.passed
-            # Freshness is expected to fail: the synthetic series ends in 2024.
             and r.check.name != "data_freshness"
         ]
         assert not critical_failures, f"Unexpected critical failures: {critical_failures}"

tests/test_style.py

@@ -8,14 +8,6 @@ import pytest
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 
-# Declared by Unicode codepoint rather than as literals, so this file does not
-# itself contain the characters it forbids. (It did, and failed its own check.)
-#
-#   U+2014 em dash        U+2013 en dash       U+2012 figure dash
-#   U+2015 horizontal bar U+2010 hyphen        U+2011 non-breaking hyphen
-#   U+2212 minus sign
-#
-# The project uses the plain ASCII hyphen everywhere instead.
 FORBIDDEN_DASHES = "".join(
     chr(code) for code in (0x2014, 0x2013, 0x2012, 0x2015, 0x2010, 0x2011, 0x2212)
 )

tests/test_warehouse.py

@@ -111,7 +111,6 @@ def test_forecast_accuracy_table_computes_error(con):
         FROM fact_forecast_accuracy
     """).df().iloc[0]
     assert row["n"] > 1000
-    # The synthetic benchmark carries roughly 2.1 percent noise by construction.
     assert 0.5 < row["mape"] < 6.0