diff --git a/.gitattributes b/.gitattributes index 4d442add34428cbfd64f33f89ebcdbf101297d95..938beb74432fa229a121b92060bcbe496c9e06af 100644 --- a/.gitattributes +++ b/.gitattributes @@ -219,3 +219,4 @@ datasets/c3s_seasonal/hindcast_month08.nc filter=lfs diff=lfs merge=lfs -text datasets/meteofrance_precipitation/daily_precipitation.csv filter=lfs diff=lfs merge=lfs -text datasets/single_trace_H605641201_2026-01-01.png filter=lfs diff=lfs merge=lfs -text datasets/spatiotemporal_gnn/selected_subgraph.png filter=lfs diff=lfs merge=lfs -text +scripts/training/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text diff --git a/scripts/__pycache__/compute_edge_width.cpython-311.pyc b/scripts/__pycache__/compute_edge_width.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66e973274cb4790441e6a11ae4add135c4fc3b87 Binary files /dev/null and b/scripts/__pycache__/compute_edge_width.cpython-311.pyc differ diff --git a/scripts/__pycache__/diagnose_persistence_baseline.cpython-311.pyc b/scripts/__pycache__/diagnose_persistence_baseline.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a48877327592ebd0879ac9af3b430f8ff67767e Binary files /dev/null and b/scripts/__pycache__/diagnose_persistence_baseline.cpython-311.pyc differ diff --git a/scripts/__pycache__/download_c3s_seasonal.cpython-311.pyc b/scripts/__pycache__/download_c3s_seasonal.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a34509e0a7a3e80324af571dc6027c7c5ab0911 Binary files /dev/null and b/scripts/__pycache__/download_c3s_seasonal.cpython-311.pyc differ diff --git a/scripts/__pycache__/fetch_ecmwf_forecast.cpython-311.pyc b/scripts/__pycache__/fetch_ecmwf_forecast.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d268d491c3faaaa3e2b1705a88c8bf868bc285d Binary files /dev/null and b/scripts/__pycache__/fetch_ecmwf_forecast.cpython-311.pyc differ diff --git a/scripts/__pycache__/fetch_eure_risle_stations.cpython-311.pyc b/scripts/__pycache__/fetch_eure_risle_stations.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8c3e9c1b66c0ae3d459574d5b8f214211768168 Binary files /dev/null and b/scripts/__pycache__/fetch_eure_risle_stations.cpython-311.pyc differ diff --git a/scripts/__pycache__/fetch_previous_runs.cpython-311.pyc b/scripts/__pycache__/fetch_previous_runs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d7eadb7cfb233ba51ea6ebf3a69464794a576bb Binary files /dev/null and b/scripts/__pycache__/fetch_previous_runs.cpython-311.pyc differ diff --git a/scripts/__pycache__/identify_flood_events.cpython-311.pyc b/scripts/__pycache__/identify_flood_events.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38bd1046599fc303ac677bd1587a112da44ec992 Binary files /dev/null and b/scripts/__pycache__/identify_flood_events.cpython-311.pyc differ diff --git a/scripts/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc b/scripts/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f139d9c21d7df245bee680216595c856f120e5d Binary files /dev/null and b/scripts/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc differ diff --git a/scripts/__pycache__/validate_forecast_skill.cpython-311.pyc b/scripts/__pycache__/validate_forecast_skill.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d26273fb3e8b6958fd9fbdcaa4915961a29e4d3 Binary files /dev/null and b/scripts/__pycache__/validate_forecast_skill.cpython-311.pyc differ diff --git a/scripts/analysis/__pycache__/check_cavity_proximity.cpython-311.pyc b/scripts/analysis/__pycache__/check_cavity_proximity.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff6ace371a8d2eed91f0a70f7f72b7c3ccf43e39 Binary files /dev/null and b/scripts/analysis/__pycache__/check_cavity_proximity.cpython-311.pyc differ diff --git a/scripts/analysis/__pycache__/check_forecast_lead5_anomaly.cpython-311.pyc b/scripts/analysis/__pycache__/check_forecast_lead5_anomaly.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd716ba91b256a3aa7b8fce88e17c16e0ec21f9a Binary files /dev/null and b/scripts/analysis/__pycache__/check_forecast_lead5_anomaly.cpython-311.pyc differ diff --git a/scripts/analysis/__pycache__/trace_forecast_lookup.cpython-311.pyc b/scripts/analysis/__pycache__/trace_forecast_lookup.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..714450fc553a60f192fffb96093de8646673144b Binary files /dev/null and b/scripts/analysis/__pycache__/trace_forecast_lookup.cpython-311.pyc differ diff --git a/scripts/analysis/analyze_bdtopo_hydro.py b/scripts/analysis/analyze_bdtopo_hydro.py new file mode 100644 index 0000000000000000000000000000000000000000..a47ac42c3e8d230384489bfd50a98b7c2fa8331b --- /dev/null +++ b/scripts/analysis/analyze_bdtopo_hydro.py @@ -0,0 +1,245 @@ +""" +Analyze the BD TOPO hydrography pulled by download_bdtopo_hydro.py: + +1. check_karst_near_betoire() -- does surface_hydrographique actually + have a karst-classified feature near the amont/aval bétoire stations + (H605641101 / H605641201)? Pure Python, no geopandas needed, so this + part works immediately regardless of what's installed. + +2. export_real_centerline() -- replace our approximate digitized + centerline (traced from a road-map screenshot, ~4km georeferencing + error) with the real troncon_hydrographique geometry for a named + river, properly ordered. Needs geopandas + shapely. + +Usage: + python analyze_bdtopo_hydro.py --check-karst + python analyze_bdtopo_hydro.py --export-centerline "Risle" --output datasets/centerlines/risle_centerline.csv +""" +import argparse +import json +import math +from pathlib import Path + +# Known amont/aval bétoire station coordinates (from station_list.csv) +BETOIRE_STATIONS = { + "H605641101": ("Ajou [amont bétoire]", 48.98492, 0.78902), + "H605641201": ("Grosley-sur-Risle [aval bétoire]", 49.04707, 0.79984), +} + + +def _haversine_km(lat1, lon1, lat2, lon2) -> float: + R = 6371.0 + lat1, lon1, lat2, lon2 = map(math.radians, [lat1, lon1, lat2, lon2]) + dlat, dlon = lat2 - lat1, lon2 - lon1 + a = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2 + return R * 2 * math.asin(math.sqrt(a)) + + +def _feature_min_distance_km(feature: dict, lat: float, lon: float) -> float: + """Minimum distance from (lat, lon) to any vertex of a GeoJSON + LineString/MultiLineString/Point feature. Vertex-level, not true + point-to-line distance, but plenty precise at this scale given + typical vertex spacing.""" + geom = feature.get("geometry") or {} + gtype = geom.get("type") + coords = geom.get("coordinates") + if coords is None: + return float("inf") + + def flatten(c, depth): + if depth == 0: + yield c + else: + for sub in c: + yield from flatten(sub, depth - 1) + + depth = {"Point": 0, "LineString": 1, "MultiLineString": 2, + "Polygon": 2, "MultiPolygon": 3}.get(gtype) + if depth is None: + return float("inf") + + best = float("inf") + for pt in flatten(coords, depth): + plon, plat = pt[0], pt[1] + d = _haversine_km(lat, lon, plat, plon) + if d < best: + best = d + return best + + +def check_karst_near_betoire( + surface_hydro_path: Path, + search_radius_km: float = 3.0, +) -> None: + """ + Search surface_hydrographique.geojson for any feature whose `Nature` + property mentions karst, within `search_radius_km` of either bétoire + station. Prints a clear verdict either way -- this is the check that + actually answers whether the bétoire naming is backed by a mapped + feature or just a station-naming convention. + """ + data = json.loads(surface_hydro_path.read_text()) + features = data.get("features", []) + print(f"Loaded {len(features)} feature(s) from {surface_hydro_path}") + + karst_features = [] + for f in features: + nature = (f.get("properties") or {}).get("nature") or (f.get("properties") or {}).get("Nature") or "" + if "karst" in str(nature).lower(): + karst_features.append(f) + + print(f"Features with 'karst' in Nature: {len(karst_features)}") + if not karst_features: + print() + print("VERDICT: No karst-classified surface_hydrographique feature found " + "anywhere in the downloaded area. Either this stretch isn't tagged " + "as karst in BD TOPO (the 'bétoire' naming may be informal/historical " + "rather than a currently-mapped classification), or the Nature " + "property uses different wording than expected -- worth printing a " + "few sample Nature values to check (see below).") + sample_natures = {(f.get("properties") or {}).get("nature") or (f.get("properties") or {}).get("Nature") + for f in features[:200]} + print(f"Sample Nature values seen in this file: {sorted(str(n) for n in sample_natures if n)[:20]}") + return + + print() + for f in karst_features: + props = f.get("properties", {}) + for code, (name, lat, lon) in BETOIRE_STATIONS.items(): + dist = _feature_min_distance_km(f, lat, lon) + flag = " <-- WITHIN SEARCH RADIUS" if dist <= search_radius_km else "" + print(f" Karst feature {props.get('nature', props.get('Nature'))!r} " + f"is {dist:.2f} km from {code} ({name}){flag}") + + close = [f for f in karst_features + for code, (name, lat, lon) in BETOIRE_STATIONS.items() + if _feature_min_distance_km(f, lat, lon) <= search_radius_km] + print() + if close: + print(f"VERDICT: CONFIRMED -- {len(close)} karst-classified feature(s) found " + f"within {search_radius_km} km of the bétoire stations. The station " + f"naming is backed by an actual mapped karst feature, not just " + f"historical naming convention.") + else: + nearest = min( + (_feature_min_distance_km(f, lat, lon), code) + for f in karst_features + for code, (name, lat, lon) in BETOIRE_STATIONS.items() + ) + print(f"VERDICT: Karst features exist in this area, but the nearest one to " + f"a bétoire station is {nearest[0]:.2f} km away (from {nearest[1]}) -- " + f"outside the {search_radius_km} km search radius. Consider widening " + f"--search-radius-km if this still seems plausibly related.") + + +def export_real_centerline( + troncon_path: Path, + river_name: str, + output_path: Path, + name_field_candidates=("cpx_toponyme_de_cours_d_eau", "toponyme", "nom_cours_d_eau"), +) -> None: + """ + Filter troncon_hydrographique to the tronçons belonging to a named + river, merge and order them into a single sequence, and write out a + centerline CSV in the same [seq, longitude, latitude] shape as our + digitized centerlines -- so this can be a drop-in replacement with + real metric-precision geometry instead of the ~4km-accurate traced + version. + + Requires geopandas + shapely. + """ + try: + import geopandas as gpd + import networkx as nx + except ImportError as e: + raise ImportError("export_real_centerline needs geopandas and networkx: " + "pip install geopandas networkx") from e + + gdf = gpd.read_file(troncon_path) + print(f"Loaded {len(gdf)} tronçons from {troncon_path}") + print(f"Available columns: {list(gdf.columns)}") + + name_field = next((c for c in name_field_candidates if c in gdf.columns), None) + if name_field is None: + raise ValueError( + f"None of {name_field_candidates} found in columns. " + f"Inspect gdf.columns and pass the right one manually." + ) + + river_gdf = gdf[gdf[name_field].astype(str).str.contains(river_name, case=False, na=False)] + print(f"Matched {len(river_gdf)} tronçon(s) for river name '{river_name}' " + f"(field: {name_field})") + if river_gdf.empty: + sample = gdf[name_field].dropna().unique()[:20] + print(f"No matches. Sample values in '{name_field}': {list(sample)}") + return + + # Build a graph from tronçon endpoints to find the correct traversal + # order (a tronçon's own row order in the file is not guaranteed to + # follow the river's course -- same lesson as station_list.csv). + G = nx.Graph() + for _, row in river_gdf.iterrows(): + coords = [(c[0], c[1]) for c in row.geometry.coords] # drop Z (altitude) if present + start, end = coords[0], coords[-1] + G.add_edge(start, end, coords=coords) + + if G.number_of_nodes() == 0: + print("No valid geometry found.") + return + + components = list(nx.connected_components(G)) + if len(components) > 1: + print(f"Warning: river tronçons form {len(components)} disconnected " + f"component(s) -- using the largest one ({max(len(c) for c in components)} nodes). " + f"This can happen at basin edges or with name-matching gaps.") + main_component = max(components, key=len) + subG = G.subgraph(main_component) + + # Walk the longest path through the component (same double-BFS + # technique used for the digitized centerline extraction). + start_node = next(iter(subG.nodes)) + lengths = nx.single_source_shortest_path_length(subG, start_node) + a = max(lengths, key=lengths.get) + lengths2 = nx.single_source_shortest_path_length(subG, a) + b = max(lengths2, key=lengths2.get) + node_path = nx.shortest_path(subG, a, b) + + all_coords = [] + for u, v in zip(node_path[:-1], node_path[1:]): + edge_coords = subG.edges[u, v]["coords"] + if edge_coords[0] != u: + edge_coords = list(reversed(edge_coords)) + all_coords.extend(edge_coords if not all_coords else edge_coords[1:]) + + import pandas as pd + result = pd.DataFrame(all_coords, columns=["longitude", "latitude"]) + result.insert(0, "seq", range(len(result))) + output_path.parent.mkdir(parents=True, exist_ok=True) + result.to_csv(output_path, index=False) + print(f"Saved {len(result)}-point real centerline for '{river_name}' to {output_path}") + print("This is real metric-precision BD TOPO geometry -- replace the old " + "digitized version by pointing centerline_dir at this file.") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Analyze BD TOPO hydrography") + parser.add_argument("--data-dir", type=Path, default=Path("datasets/bdtopo_hydro")) + parser.add_argument("--check-karst", action="store_true") + parser.add_argument("--search-radius-km", type=float, default=3.0) + parser.add_argument("--export-centerline", type=str, default=None, + help="River name to extract, e.g. 'Risle' or 'Eure'") + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args() + + if args.check_karst: + check_karst_near_betoire(args.data_dir / "surface_hydrographique.geojson", + args.search_radius_km) + + if args.export_centerline: + out = args.output or Path(f"datasets/centerlines/{args.export_centerline.lower()}_centerline_real.csv") + export_real_centerline(args.data_dir / "troncon_hydrographique.geojson", + args.export_centerline, out) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/analysis/check_cavity_proximity.py b/scripts/analysis/check_cavity_proximity.py new file mode 100644 index 0000000000000000000000000000000000000000..7425cbf158572e84a3a36a34a7ddcfb31e6ac11c --- /dev/null +++ b/scripts/analysis/check_cavity_proximity.py @@ -0,0 +1,169 @@ +""" +Real, independent karst cross-check for every real gauge station, using +the actual raw BDCavités data -- not the precomputed distance_to_ +nearest_cavity_km column, which turned out to never have been saved +anywhere in the current reach-graph pipeline (confirmed directly: the +column is absent from both real enriched node files, despite the raw +cavité data itself existing on disk). + +Built specifically because the current karst/losing-reach flag +(verified_continuous=False on 3 real edges of La Risle's bétoire +stretch) is a hardcoded list based on real station naming evidence +("[amont bétoire]"/"[aval bétoire]"), not a systematic scan -- it can +only ever confirm what was already suspected from names, never surface +a DIFFERENT real station that also sits near a real cavity but never +got a "bétoire"-style name. This script is that systematic scan. + +Usage: + python -m scripts.analysis.check_cavity_proximity --data-root datasets +""" +import argparse +import json +from pathlib import Path +from typing import List, Optional + +import numpy as np +import pandas as pd + + +def _haversine_km_vec(lat1, lon1, lat2, lon2): + """Real haversine distance, vectorized -- same formula used + throughout this project's other real distance computations.""" + R = 6371.0 + lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2]) + dlat, dlon = lat2 - lat1, lon2 - lon1 + a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2 + return R * 2 * np.arcsin(np.sqrt(a)) + + +def load_real_cavity_coordinates(geojson_path: Path, allowed_types: Optional[List[str]] = None) -> np.ndarray: + """ + Extracts real [lat, lon] pairs directly from the raw BDCavités + GeoJSON -- point features only, skipping anything malformed rather + than crashing the whole load on a single bad real record (large + real government datasets do have occasional genuine data-quality + issues; skip-and-count is more useful here than a hard failure). + + allowed_types, when given, filters on the real "type_cavite" field. + Confirmed as a real, necessary filter, not an optional refinement: + an unfiltered first run showed 24 of 27 real stations within 1km of + *some* cavity, with counts in the thousands -- because BDCavités + catalogs every cavity type, and the real breakdown (confirmed + directly against this project's actual file) is dominated by + quarries (carrière, 21,813), civil/military structures (ouvrage + civil/militaire, ~9,900), and cellars (cave, 3,990) -- none + hydrologically meaningful. Only "naturelle" (8,313) corresponds to + genuine natural cavities, the category that could actually indicate + real karst/water-loss connectivity. "indéterminé" (18,349) is + genuinely ambiguous -- could hide real unclassified natural + features or unclassified quarries -- so it's deliberately not + included by default; check separately with it included rather than + assume either way. + """ + with open(geojson_path) as f: + data = json.load(f) + + coords = [] + n_skipped = 0 + n_filtered = 0 + for feature in data.get("features", []): + geom = feature.get("geometry") + if geom is None or geom.get("type") != "Point": + n_skipped += 1 + continue + if allowed_types is not None: + real_type = feature.get("properties", {}).get("type_cavite") + if real_type not in allowed_types: + n_filtered += 1 + continue + lon, lat = geom["coordinates"][0], geom["coordinates"][1] + coords.append((lat, lon)) + + filter_msg = f", {n_filtered} filtered out by type" if allowed_types is not None else "" + print(f"Loaded {len(coords)} real cavity point(s) from {geojson_path.name} " + f"({n_skipped} non-point/malformed feature(s) skipped{filter_msg})") + return np.array(coords) + + +def check_cavity_proximity(stations_df: pd.DataFrame, cavity_coords: np.ndarray, radius_km: float = 20.0) -> pd.DataFrame: + """ + Real nearest-cavity distance and real cavity count within radius_km, + per real station -- same two metrics documented for the (never + actually saved) distance_to_nearest_cavity_km / n_cavities_within_ + 20km features, computed fresh here directly against the real raw + data. + """ + cav_lat, cav_lon = cavity_coords[:, 0], cavity_coords[:, 1] + results = [] + for _, row in stations_df.iterrows(): + dists = _haversine_km_vec(row["latitude"], row["longitude"], cav_lat, cav_lon) + nearest_km = float(dists.min()) + n_within_radius = int((dists <= radius_km).sum()) + results.append({ + "station_code": row["station_code"], + "distance_to_nearest_cavity_km": nearest_km, + "n_cavities_within_20km": n_within_radius, + }) + return pd.DataFrame(results).sort_values("distance_to_nearest_cavity_km") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Real cavity-proximity cross-check for every real gauge station") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None) + parser.add_argument("--close-threshold-km", type=float, default=1.0, + help="Real distance below which a station is flagged as plausibly karst-connected.") + args = parser.parse_args() + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + stations_df = pd.read_csv(stations_path) + + geojson_path = args.data_root / "bdcavites" / "cavite_localisee.geojson" + if not geojson_path.exists(): + print(f"No real cavity file found at {geojson_path} -- run " + f"scripts.data_acquisition.download_bdcavites first.") + return + + # Two real, confirmed-necessary filtered runs, not one -- an + # unfiltered check showed 24/27 real stations within 1km of *some* + # cavity (counts in the thousands), because BDCavités catalogs + # every cavity type and quarries/civil structures/cellars dominate + # the real breakdown. "naturelle" is the conservative, defensible + # filter; "naturelle + indéterminé" is a broader one that also + # captures any real natural feature that was never properly typed + # -- reported side by side rather than picking one and hiding the + # other's disagreement. + for label, allowed_types in [ + ("naturelle only (conservative)", ["naturelle"]), + ("naturelle + indéterminé (broader)", ["naturelle", "indéterminé"]), + ]: + print() + print("=" * 70) + print(f"Filter: {label}") + print("=" * 70) + cavity_coords = load_real_cavity_coordinates(geojson_path, allowed_types=allowed_types) + if len(cavity_coords) == 0: + print("Zero real cavities matched this filter -- nothing to check against.") + continue + + result_df = check_cavity_proximity(stations_df, cavity_coords) + print(result_df.to_string(index=False)) + + close = result_df[result_df["distance_to_nearest_cavity_km"] < args.close_threshold_km] + print() + if len(close) > 0: + print(f"Station(s) within {args.close_threshold_km}km of a real, filtered cavity " + f"(cross-check against the known bétoire stations -- H605641101, H605022010, " + f"H605641401, H605641201 -- anything else here is a real, new candidate):") + print(close.to_string(index=False)) + else: + print(f"No real station within {args.close_threshold_km}km of a real, filtered cavity.") + + output_path = args.data_root / f"cavity_proximity_check_{allowed_types[0]}.csv" + result_df.to_csv(output_path, index=False) + print(f"Saved to {output_path}") + print(f"\nSaved to {output_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/analysis/check_forecast_lead5_anomaly.py b/scripts/analysis/check_forecast_lead5_anomaly.py new file mode 100644 index 0000000000000000000000000000000000000000..5a37c54c619a074bfe840a54e453072208a6ae7a --- /dev/null +++ b/scripts/analysis/check_forecast_lead5_anomaly.py @@ -0,0 +1,120 @@ +""" +Direct check of the raw previous_runs_forecast.csv archive, independent +of the model/evaluation pipeline -- built specifically because +evaluate_flood_detection.py's --dump-fp-details showed something +systematic and graph-wide at the 5-day horizon (185/185 real test +examples produced at least one real false positive there, often +several stations simultaneously, including stations on different +rivers with no surface connection). --check-forecast-fp-correlation +already ruled out forecast AVAILABILITY as the driver; this checks +whether the real forecast VALUES themselves are anomalous specifically +at lead time 5, compared to the other real lead times in the same file. + +Usage: + python -m scripts.analysis.check_forecast_lead5_anomaly --data-root datasets +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +FORECAST_LEAD_TIMES = [1, 2, 3, 5, 7] # matches dynamic_features.py's real, confirmed coverage + + +def summarize_lead_time_column(df: pd.DataFrame, lead_time: int) -> dict: + """ + Real summary statistics for one real precipitation_previous_dayN + column -- real coverage, real central tendency, real spread, and + explicit counts of two real, concrete anomaly types (negative + precipitation, which is physically impossible, and implausibly + extreme values) rather than just eyeballing a mean/max pair. + """ + col = f"precipitation_previous_day{lead_time}" + if col not in df.columns: + return {"lead_time": lead_time, "column_present": False} + + values = df[col].dropna() + n_negative = int((values < 0).sum()) + # A real, extreme daily precipitation threshold -- the wettest days + # ever recorded anywhere are on the order of a few hundred mm; used + # here only to flag values worth a second look, not as a hard + # physical cutoff. + n_extreme = int((values > 300).sum()) + + return { + "lead_time": lead_time, + "column_present": True, + "n_real": len(values), + "mean": float(values.mean()) if len(values) else float("nan"), + "median": float(values.median()) if len(values) else float("nan"), + "std": float(values.std()) if len(values) else float("nan"), + "min": float(values.min()) if len(values) else float("nan"), + "max": float(values.max()) if len(values) else float("nan"), + "n_negative": n_negative, + "n_extreme_gt_300mm": n_extreme, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check the real ECMWF forecast archive for a lead-time-5-specific anomaly") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--forecast-path", type=Path, default=None, + help="Defaults to /previous_runs_forecast.csv") + args = parser.parse_args() + + forecast_path = args.forecast_path or (args.data_root / "previous_runs_forecast.csv") + if not forecast_path.exists(): + print(f"No real forecast archive found at {forecast_path}.") + return + + df = pd.read_csv(forecast_path) + print(f"Loaded {len(df)} real row(s) from {forecast_path}") + print() + + rows = [] + for lead_time in FORECAST_LEAD_TIMES: + summary = summarize_lead_time_column(df, lead_time) + rows.append(summary) + + result_df = pd.DataFrame(rows) + print("=" * 70) + print("Real forecast precipitation, by lead time (mm)") + print("=" * 70) + print(result_df.to_string(index=False)) + + present = result_df[result_df["column_present"]] + if len(present) < 2: + print("\nFewer than 2 real lead-time columns present -- nothing to compare.") + return + + if 5 in present["lead_time"].values: + lead5 = present[present["lead_time"] == 5].iloc[0] + others = present[present["lead_time"] != 5] + others_mean_of_means = others["mean"].mean() + others_mean_of_maxes = others["max"].mean() + print(f"\nLead time 5 mean ({lead5['mean']:.2f}mm) vs. real average of other lead times' means " + f"({others_mean_of_means:.2f}mm)") + print(f"Lead time 5 max ({lead5['max']:.2f}mm) vs. real average of other lead times' maxes " + f"({others_mean_of_maxes:.2f}mm)") + print(f"Lead time 5 negative values: {int(lead5['n_negative'])}; extreme (>300mm) values: " + f"{int(lead5['n_extreme_gt_300mm'])}") + + mean_ratio = lead5["mean"] / others_mean_of_means if others_mean_of_means > 0 else float("inf") + max_ratio = lead5["max"] / others_mean_of_maxes if others_mean_of_maxes > 0 else float("inf") + if mean_ratio > 2.0 or max_ratio > 2.0 or lead5["n_negative"] > 0: + print(f"\nLead time 5 shows a real, meaningful anomaly relative to the other real lead times " + f"(mean ratio {mean_ratio:.2f}x, max ratio {max_ratio:.2f}x) -- real, direct evidence " + f"worth investigating further as the actual cause of the day-5 false-positive pattern.") + else: + print(f"\nLead time 5 does NOT show a meaningful anomaly relative to the other real lead " + f"times (mean ratio {mean_ratio:.2f}x, max ratio {max_ratio:.2f}x) -- the raw forecast " + f"values themselves aren't the explanation; the real cause likely lies in how this " + f"lead time's data is being consumed downstream (e.g. the anchor/target-date mapping, " + f"or the model's own horizon-conditioning for h=5 specifically), not the raw archive.") + else: + print("\nLead time 5 column not present in this real file -- can't run the real comparison.") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/cross_check_catchments.py b/scripts/analysis/cross_check_catchments.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f26892312abb4b975d8da449619806294e15a9 --- /dev/null +++ b/scripts/analysis/cross_check_catchments.py @@ -0,0 +1,50 @@ +""" +Spatially match each station to the catchment polygon that actually +contains it (point-in-polygon), rather than guessing at a shared ID +field across BD TOPO and Hub'Eau. This is the correct approach when +two datasets don't share a clean join key. +""" +import sys +sys.path.insert(0, ".") +from pathlib import Path +import pandas as pd +import geopandas as gpd + +from src.data.loaders.bdtopo_hydro import BDTopoHydroLoader + +loader = BDTopoHydroLoader(data_path=Path("datasets/bdtopo_hydro")) +catchments = loader.load_catchments() +catchments_metric = catchments.to_crs(epsg=2154) # Lambert-93 for accurate area in meters +catchments["area_km2"] = catchments_metric.geometry.area / 1e6 + +stations = pd.read_csv("datasets/station_elevations.csv") +hubeau = pd.read_csv("datasets/catchment_area.csv").dropna(subset=["catchment_area_km2"]) + +stations_gdf = gpd.GeoDataFrame( + stations, + geometry=gpd.points_from_xy(stations["longitude"], stations["latitude"]), + crs="EPSG:4326", +).to_crs(catchments.crs) + +# Spatial join: which catchment polygon contains each station? +joined = gpd.sjoin(stations_gdf, catchments[["area_km2", "toponyme", "geometry"]], + how="left", predicate="within") + +result = joined[["station_code", "toponyme", "area_km2"]].rename( + columns={"area_km2": "bdtopo_area_km2", "toponyme": "bdtopo_catchment_name"} +) +result = result.merge(hubeau[["station_code", "catchment_area_km2"]], on="station_code", how="left") +result = result.rename(columns={"catchment_area_km2": "hubeau_area_km2"}) +result["ratio"] = result["bdtopo_area_km2"] / result["hubeau_area_km2"] + +print(result.to_string(index=False)) +print() +matched = result.dropna(subset=["bdtopo_area_km2", "hubeau_area_km2"]) +print(f"Stations with BOTH a containing BD TOPO polygon AND a Hub'Eau value: {len(matched)}") +if len(matched) > 0: + print(f"Area ratio (BD TOPO / Hub'Eau) — mean: {matched['ratio'].mean():.2f}, " + f"median: {matched['ratio'].median():.2f}") + print("(A ratio near 1.0 across most stations would confirm both sources agree; " + "large/inconsistent ratios would suggest the polygons represent something " + "other than the gauge's own catchment -- e.g. a downstream aggregate, or " + "no polygon actually contains that point due to bbox edge effects.)") \ No newline at end of file diff --git a/scripts/analysis/run_bdtopo_checks.py b/scripts/analysis/run_bdtopo_checks.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ca6dbd4ca6b056d8bba23a748840476e7bc676 --- /dev/null +++ b/scripts/analysis/run_bdtopo_checks.py @@ -0,0 +1,71 @@ +""" +Run this against your real downloaded BD TOPO data to: +1. Actually answer the karst/bétoire question with real data. +2. Cross-validate the new catchment polygons against Hub'Eau's surface_bv + numbers (from catchment_area.csv, built earlier via download_catchment_area.py). +""" +import sys +sys.path.insert(0, ".") +from pathlib import Path +from src.data.loaders.bdtopo_hydro import BDTopoHydroLoader + +loader = BDTopoHydroLoader(data_path=Path("datasets/bdtopo_hydro")) + +# --- 1. The karst question, for real this time --- +print("=" * 70) +print("KARST CHECK near amont/aval bétoire stations") +print("=" * 70) +BETOIRE_STATIONS = { + "H605641101": ("Ajou [amont bétoire]", 48.98492, 0.78902), + "H605641201": ("Grosley-sur-Risle [aval bétoire]", 49.04707, 0.79984), +} +surfaces = loader.load_surfaces() +any_found = False +for code, (name, lat, lon) in BETOIRE_STATIONS.items(): + results = loader.check_karst_near_point(lat, lon, search_radius_km=3.0, surfaces_geojson=surfaces) + print(f"\n{code} ({name}):") + if results: + any_found = True + for r in results: + print(f" -> {r['nature']!r} at {r['distance_km']:.2f} km") + else: + print(" -> no karst-classified feature within 3 km") + +print() +if any_found: + print("VERDICT: CONFIRMED by real BD TOPO data -- the bétoire naming is backed " + "by an actual mapped karst feature.") +else: + print("VERDICT: No karst feature found within 3km. Try a wider search radius, " + "or this may just be historical/informal naming not reflected in the " + "current BD TOPO classification.") + +# --- 2. Cross-validate catchment polygons against Hub'Eau surface_bv --- +print() +print("=" * 70) +print("CATCHMENT POLYGON CROSS-CHECK vs Hub'Eau surface_bv") +print("=" * 70) +try: + import geopandas as gpd + import pandas as pd + + catchments = loader.load_catchments() + print(f"Loaded {len(catchments)} catchment polygon(s)") + print(f"Columns: {list(catchments.columns)}") + + hubeau_path = Path("datasets/catchment_area.csv") + if hubeau_path.exists(): + hubeau = pd.read_csv(hubeau_path).dropna(subset=["catchment_area_km2"]) + print(f"\nLoaded {len(hubeau)} Hub'Eau reference values from {hubeau_path}") + # Compute polygon areas in km^2 (reproject to a metric CRS for accurate area) + catchments_metric = catchments.to_crs(epsg=2154) # Lambert-93, standard for France + catchments["area_km2_computed"] = catchments_metric.geometry.area / 1e6 + print(catchments[["area_km2_computed"]].describe()) + print("\n(Match these against datasets/catchment_area.csv's real Hub'Eau values " + "by whichever ID field the polygons carry -- check catchments.columns " + "above for a station/site code to join on.)") + else: + print(f"\n{hubeau_path} not found -- skipping cross-check. " + f"(Run download_catchment_area.py first if you want this comparison.)") +except ImportError: + print("geopandas not installed -- skipping catchment polygon check.") \ No newline at end of file diff --git a/scripts/analysis/trace_forecast_lookup.py b/scripts/analysis/trace_forecast_lookup.py new file mode 100644 index 0000000000000000000000000000000000000000..d22a66f6234edaa6b82c4d1a420c9c32e7d2bd81 --- /dev/null +++ b/scripts/analysis/trace_forecast_lookup.py @@ -0,0 +1,127 @@ +""" +Directly traces a handful of real false positives (from evaluate_flood_ +detection.py's --dump-fp-details output) through build_forecast_tensor's +actual real lookup logic, by hand, to check for a real date-convention +mismatch -- built after noticing something concrete while investigating +the day-5 false-positive pattern: + +Every other real place in this project that maps a horizon h to a real +calendar date uses target_date = anchor + (h - 1) -- confirmed directly +in both prepare_graph_training_windows (the real training-target +construction) and evaluate_flood_detection.py's own confusion-matrix +loop. But build_forecast_tensor uses target_date = anchor + h (no "-1"). + +This means: for horizon h, the real target this project is actually +predicting is at anchor + (h-1), but the forecast value FED INTO the +model for that same horizon slot is looked up at anchor + h -- one +real calendar day later than the thing actually being predicted. This +script shows exactly what real forecast row/column gets retrieved for +real (anchor, h=5) pairs pulled from an actual false-positive dump, so +the real size and direction of this mismatch is visible directly, +not just asserted. + +Usage: + python -m scripts.analysis.trace_forecast_lookup --data-root datasets \ + --fp-details datasets/fp_details_horizon_5.csv --horizon 5 +""" +import argparse +from pathlib import Path + +import pandas as pd + + +def trace_one(anchor: pd.Timestamp, station_code: str, horizon: int, df_indexed: pd.DataFrame) -> dict: + """ + Mirrors build_forecast_tensor's real lookup exactly -- same + target_date/column formula, same real row lookup -- so this shows + precisely what that function actually retrieves for this real + (anchor, horizon) pair, not an approximation of it. + """ + real_target_date = anchor + pd.Timedelta(days=horizon - 1) # what's ACTUALLY being predicted + forecast_lookup_date = anchor + pd.Timedelta(days=horizon) # what build_forecast_tensor ACTUALLY queries + col = f"precipitation_previous_day{horizon}" + + result = { + "anchor": anchor.date().isoformat(), + "real_target_date": real_target_date.date().isoformat(), + "forecast_lookup_date": forecast_lookup_date.date().isoformat(), + "date_mismatch_days": (forecast_lookup_date - real_target_date).days, + "station_code": station_code, + "column": col, + } + try: + value = df_indexed.loc[(forecast_lookup_date, station_code), col] + result["forecast_value_actually_used"] = float(value) if pd.notna(value) else None + except KeyError: + result["forecast_value_actually_used"] = None + + # What the forecast value WOULD have been if looked up at the real + # target date instead, for direct comparison -- not necessarily a + # "correct" value (that date's own previous_day{h} column reflects + # a different, (h) days in advance forecast, not h-1), just useful + # to see whether it's meaningfully different from what's actually used. + try: + value_at_real_target = df_indexed.loc[(real_target_date, station_code), col] + result["value_at_real_target_date_same_column"] = float(value_at_real_target) if pd.notna(value_at_real_target) else None + except KeyError: + result["value_at_real_target_date_same_column"] = None + + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description="Trace real forecast lookups for a handful of real false positives") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--forecast-path", type=Path, default=None, + help="Defaults to /previous_runs_forecast.csv") + parser.add_argument("--fp-details", type=Path, required=True, + help="Path to a real fp_details_horizon_N.csv from evaluate_flood_detection.py's --dump-fp-details") + parser.add_argument("--horizon", type=int, required=True) + parser.add_argument("--n-samples", type=int, default=10) + args = parser.parse_args() + + forecast_path = args.forecast_path or (args.data_root / "previous_runs_forecast.csv") + if not forecast_path.exists(): + print(f"No real forecast archive found at {forecast_path}.") + return + if not args.fp_details.exists(): + print(f"No real false-positive detail file found at {args.fp_details}.") + return + + forecast_df = pd.read_csv(forecast_path) + forecast_df["date"] = pd.to_datetime(forecast_df["date"]) + df_indexed = forecast_df.set_index(["date", "station_code"]) + + fp_df = pd.read_csv(args.fp_details) + fp_df["target_date"] = pd.to_datetime(fp_df["target_date"]) + # fp_details' own target_date IS the real target date (h-1 + # convention, matching evaluate_flood_detection.py's own confusion- + # matrix loop) -- recover the real anchor from it directly. + fp_df["anchor"] = fp_df["target_date"] - pd.Timedelta(days=args.horizon - 1) + + sample = fp_df.head(args.n_samples) + print(f"Tracing {len(sample)} real false positive(s) at horizon={args.horizon}d through the real " + f"forecast lookup logic\n") + + rows = [] + for _, row in sample.iterrows(): + traced = trace_one(row["anchor"], row["station_code"], args.horizon, df_indexed) + rows.append(traced) + + result_df = pd.DataFrame(rows) + print(result_df.to_string(index=False)) + + n_mismatched_dates = int((result_df["date_mismatch_days"] != 0).sum()) + print(f"\n{n_mismatched_dates}/{len(result_df)} real traced examples show a real date mismatch " + f"between what's being predicted (real_target_date) and what the forecast is actually " + f"looked up for (forecast_lookup_date).") + if n_mismatched_dates > 0: + print(f"This is a real, systematic {result_df['date_mismatch_days'].iloc[0]}-day offset (from " + f"build_forecast_tensor's target_date = anchor + h, vs. every other real place in this " + f"project using anchor + h - 1) -- the model is being conditioned on the forecast for the " + f"day AFTER what it's actually predicting at this horizon, not the forecast for the real " + f"target date itself.") + + +if __name__ == "__main__": + main() diff --git a/scripts/data_acquisition/__pycache__/fetch_meteofrance_precipitation.cpython-311.pyc b/scripts/data_acquisition/__pycache__/fetch_meteofrance_precipitation.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34169eb4f9c89fdb1c76a596c608eeb94e776054 Binary files /dev/null and b/scripts/data_acquisition/__pycache__/fetch_meteofrance_precipitation.cpython-311.pyc differ diff --git a/scripts/data_acquisition/download_bdcavites.py b/scripts/data_acquisition/download_bdcavites.py new file mode 100644 index 0000000000000000000000000000000000000000..9006cafa91d334b1e750be092f5fab30ac72efc7 --- /dev/null +++ b/scripts/data_acquisition/download_bdcavites.py @@ -0,0 +1,146 @@ +""" +Downloads BDCavités (BRGM's national underground cavity inventory -- +karst sinkholes, quarries, marl pits, natural cavities) for the Eure/ +Risle area, via Géorisques' WFS. + +Directly relevant to the bétoire investigation: this is an independent, +purpose-built dataset for exactly this phenomenon, unlike inferring it +from Hub'Eau station naming or BD TOPO's provisional karst attribute. + +CONFIRMED (fetched GetCapabilities directly): endpoint +https://georisques.gouv.fr/services, WFS 1.1.0, typeName +CAVITE_LOCALISEE ("Cavités souterraines abandonnées d'origine non +minière"), GeoJSON output supported directly. Axis order for this +specific server is NOT separately confirmed -- reusing the same +lon,lat-then-lat,lon retry that scripts/download_bdtopo_hydro.py needed +for a different WFS server, since different servers have behaved +differently on this before and there's no reason to assume this one +won't too. + +Usage: + python -m scripts.download_bdcavites --check + python -m scripts.download_bdcavites +""" +import argparse +import json +import sys +from pathlib import Path + +import requests + +WFS_URL = "https://georisques.gouv.fr/services" +TYPE_NAME = "CAVITE_LOCALISEE" + +# Same widened bbox as scripts/download_bdtopo_hydro.py's BBOX (updated +# after confirmed real evidence of truncation on the original, tighter +# box) -- keeping both scripts scoped to the same area. +BBOX = (-0.1, 47.7, 2.1, 49.9) # (min_lon, min_lat, max_lon, max_lat) + + +def check_typename() -> bool: + params = { + "SERVICE": "WFS", "VERSION": "1.1.0", "REQUEST": "GetFeature", + "TYPENAME": TYPE_NAME, "MAXFEATURES": 1, "OUTPUTFORMAT": "application/json; subtype=geojson; charset=utf-8", + } + resp = requests.get(WFS_URL, params=params, timeout=30) + ok, detail = _parse_response(resp) + print(f" {TYPE_NAME}: {'OK' if ok else 'FAILED'}") + if not ok: + print(f" {detail}") + return ok + + +def _parse_response(resp: "requests.Response"): + if resp.status_code != 200: + return False, f"HTTP {resp.status_code}: {resp.text[:400]}" + try: + data = resp.json() + except ValueError: + return False, f"Response was not JSON (likely a WFS ExceptionReport): {resp.text[:400]}" + if isinstance(data, dict) and data.get("type") == "FeatureCollection": + return True, "" + return False, f"Response was JSON but not a FeatureCollection: {str(data)[:400]}" + + +def fetch_all_pages(bbox_param: str, page_size: int = 1000) -> list: + all_features = [] + start_index = 0 + while True: + params = { + "SERVICE": "WFS", "VERSION": "1.1.0", "REQUEST": "GetFeature", + "TYPENAME": TYPE_NAME, "BBOX": bbox_param, + "OUTPUTFORMAT": "application/json; subtype=geojson; charset=utf-8", + "MAXFEATURES": page_size, "STARTINDEX": start_index, + } + resp = requests.get(WFS_URL, params=params, timeout=60) + ok, detail = _parse_response(resp) + if not ok: + print(f" ERROR at startIndex={start_index}: {detail}") + break + data = resp.json() + features = data.get("features", []) + if not features: + break + all_features.extend(features) + if len(features) < page_size: + break + start_index += page_size + return all_features + + +def fetch_with_axis_retry(bbox: tuple, out_path: Path) -> None: + min_lon, min_lat, max_lon, max_lat = bbox + orderings = [ + ("lon,lat", f"{min_lon},{min_lat},{max_lon},{max_lat},urn:ogc:def:crs:EPSG::4326"), + ("lat,lon", f"{min_lat},{min_lon},{max_lat},{max_lon},urn:ogc:def:crs:EPSG::4326"), + ] + features = [] + for label, bbox_param in orderings: + print(f" trying axis order {label}...") + features = fetch_all_pages(bbox_param) + if features: + print(f" -> {label} worked ({len(features)} feature(s))") + break + print(f" -> {label} returned 0 features") + + geojson = {"type": "FeatureCollection", "features": features} + out_path.write_text(json.dumps(geojson)) + if features: + print(f"Saved {len(features)} feature(s) to {out_path}") + else: + print(f"Saved an EMPTY file to {out_path} -- both axis orders returned nothing. " + f"Run --check first, or this area may genuinely have zero recorded cavities " + f"(plausible -- BDCavités coverage is built department-by-department and " + f"isn't uniformly complete everywhere).") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download BDCavités for the Eure/Risle area") + parser.add_argument("--check", action="store_true") + parser.add_argument("--output-dir", type=Path, default=Path("datasets/bdcavites")) + args = parser.parse_args() + + if args.check: + print("Testing typeName against the live service...") + ok = check_typename() + if not ok: + print("\nFailed -- the typeName or service details may have changed since this " + "script was written. Fetch GetCapabilities directly to check:") + print(f" {WFS_URL}?SERVICE=WFS&VERSION=1.1.0&REQUEST=GetCapabilities") + else: + print("\nOK. Re-run without --check to download.") + return + + args.output_dir.mkdir(parents=True, exist_ok=True) + print("Fetching CAVITE_LOCALISEE (BDCavités)...") + fetch_with_axis_retry(BBOX, args.output_dir / "cavite_localisee.geojson") + print() + print("Once downloaded, cross-reference against the amont/aval bétoire stations " + "(H605641101 at 48.98492,0.78902 and H605641201 at 49.04707,0.79984) the same " + "way scripts/analyze_bdtopo_hydro.py's check_karst_near_betoire did for the " + "BD TOPO karst attribute -- this is a genuinely independent second check, " + "not a re-run of the same one.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/download_bdcharm.py b/scripts/data_acquisition/download_bdcharm.py new file mode 100644 index 0000000000000000000000000000000000000000..05e43c3256ce116956d6330becf0d88b6f6c21e8 --- /dev/null +++ b/scripts/data_acquisition/download_bdcharm.py @@ -0,0 +1,107 @@ +""" +Downloads BD Charm-50 (BRGM's harmonized 1:50,000 geological maps) for +the departments covering the Eure/Risle study area. + +CONFIRMED (via data.gouv.fr / InfoTerre): free, open (Licence Ouverte), +no authentication -- direct per-department ZIP download from InfoTerre. +This is a genuinely different access pattern than the WFS sources +elsewhere in this project (BD TOPO, BDCavités): no bbox query, no axis- +order ambiguity, just a fixed URL per department. Real, working URL +pattern (found directly, not guessed): + http://infoterre.brgm.fr/telechargements/BDCharm50/GEO050K_HARM_XXX.zip +where XXX is the 3-digit department code (e.g. 027 for Eure). + +NOTE: a separate, DIFFERENT distribution of this same underlying dataset +exists that requires CIGAL network membership (seen for at least Alsace +regional data) -- that is NOT what this script uses. This script only +uses the free InfoTerre download confirmed via data.gouv.fr. + +DEPARTMENTS: 27 (Eure), 61 (Orne), 28 (Eure-et-Loir) -- 27 and 61 +confirmed directly from real INSEE codes seen in this project's own +station_list.csv (61342, 27040, 27116, 27468); 28 included because the +Eure's own southern tributaries (Voise, Drouette) run through Chartres/ +Dreux, unambiguously in Eure-et-Loir. Add more department codes via +--departments if the real geographic extent turns out to need them. + +Usage: + python -m scripts.download_bdcharm50 +""" +import argparse +import zipfile +from pathlib import Path + +import requests + +DEFAULT_DEPARTMENTS = ["027", "028", "061"] +BASE_URL = "http://infoterre.brgm.fr/telechargements/BDCharm50" + + +def download_department(dept: str, output_dir: Path) -> bool: + url = f"{BASE_URL}/GEO050K_HARM_{dept}.zip" + zip_path = output_dir / f"GEO050K_HARM_{dept}.zip" + + print(f" department {dept}: {url}") + try: + resp = requests.get(url, timeout=120, stream=True) + except requests.RequestException as e: + print(f" FAILED: {e}") + return False + + if resp.status_code != 200: + print(f" FAILED: HTTP {resp.status_code}") + return False + + content_type = resp.headers.get("Content-Type", "") + if "zip" not in content_type and "octet-stream" not in content_type: + # A 200 status with an HTML content-type here usually means an + # error page or a "department not available" page was returned + # instead of the real ZIP -- catching this explicitly rather + # than silently saving an HTML file with a .zip extension. + print(f" WARNING: Content-Type is {content_type!r}, not zip -- " + f"this department's file may not exist at this URL. Saving " + f"anyway for inspection, but verify before trusting it.") + + zip_path.write_bytes(resp.content) + size_kb = zip_path.stat().st_size / 1024 + print(f" saved {zip_path} ({size_kb:.0f} KB)") + + extract_dir = output_dir / f"dept_{dept}" + try: + with zipfile.ZipFile(zip_path) as zf: + zf.extractall(extract_dir) + shp_files = list(extract_dir.rglob("*.shp")) + print(f" extracted to {extract_dir} ({len(shp_files)} .shp file(s) found)") + return True + except zipfile.BadZipFile: + print(f" FAILED: downloaded file is not a valid zip -- likely an error page, " + f"not real data. Check {zip_path} directly.") + return False + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download BD Charm-50 geological maps") + parser.add_argument("--departments", nargs="+", default=DEFAULT_DEPARTMENTS, + help="3-digit department codes, e.g. 027 028 061") + parser.add_argument("--output-dir", type=Path, default=Path("datasets/bdcharm50")) + args = parser.parse_args() + + args.output_dir.mkdir(parents=True, exist_ok=True) + print(f"Downloading BD Charm-50 for departments: {args.departments}") + print() + + results = {} + for dept in args.departments: + results[dept] = download_department(dept, args.output_dir) + print() + + print("=" * 60) + ok = [d for d, r in results.items() if r] + failed = [d for d, r in results.items() if not r] + print(f"Succeeded: {ok}") + if failed: + print(f"Failed: {failed} -- check the URL pattern still matches by visiting " + f"https://infoterre.brgm.fr/page/telechargement-cartes-geologiques directly") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/download_bdtopo_hydro.py b/scripts/data_acquisition/download_bdtopo_hydro.py new file mode 100644 index 0000000000000000000000000000000000000000..f605db34cf4b056cf96055fdeb4ceb9f93441d82 --- /dev/null +++ b/scripts/data_acquisition/download_bdtopo_hydro.py @@ -0,0 +1,244 @@ +""" +Download BD TOPO hydrography (troncon_hydrographique, surface_hydrographique) +for the Eure/Risle area via IGN's Geoplateforme WFS, instead of pulling the +whole national dataset. + +VERIFICATION NOTE: I confirmed the WFS endpoint itself is real and live +(https://data.geopf.fr/wfs, WFS 2.0.0) and that a "BDTOPO_V3" namespace +exists on it, by fetching GetCapabilities directly. I could NOT confirm +the exact typeName for the hydrography layers within it — the full +capabilities document is huge (thousands of layers across all IGN +products) and got truncated before reaching them. DEFAULT_TYPENAMES +below is my best guess based on IGN's usual naming convention, NOT a +verified value. This script's first step (check_typenames) tests each +name with a tiny query before attempting the real extraction, and tells +you exactly how to find the right name yourself if my guess is wrong. + +Usage: + python download_bdtopo_hydro.py --check # verify typeNames first + python download_bdtopo_hydro.py # then run the real extraction +""" +import argparse +import sys +from pathlib import Path + +import requests +import pandas as pd + +WFS_URL = "https://data.geopf.fr/wfs" + +# Best-guess typeNames -- see VERIFICATION NOTE above. Override with +# --troncon-typename / --surface-typename / --catchment-typename if wrong. +# troncon/surface confirmed correct against the real service (see +# CONFIRMED note below); catchment is still an unverified guess. +DEFAULT_TRONCON_TYPENAME = "BDTOPO_V3:troncon_hydrographique" +DEFAULT_SURFACE_TYPENAME = "BDTOPO_V3:surface_hydrographique" +DEFAULT_CATCHMENT_TYPENAME = "BDTOPO_V3:bassin_versant_topographique" + +# CONFIRMED: both troncon_hydrographique and surface_hydrographique +# typeNames work against the live service (verified in this project's +# actual run: 30045 tronçons fetched successfully). bassin_versant_ +# topographique has NOT been verified the same way -- run --check +# before trusting it. + +# Bounding box around the Eure/Risle stations, in WGS84 (matches the +# lon/lat range in your real station_list.csv, with a small margin). +BBOX = (0.3, 48.3, 1.7, 49.5) # (min_lon, min_lat, max_lon, max_lat) + + +def find_hydro_typenames(keyword: str = "hydrographique") -> list: + """ + Fetch GetCapabilities and search for FeatureType entries + containing `keyword`. This is the fallback if DEFAULT_*_TYPENAME + turns out to be wrong -- run this to get the real list. + """ + print(f"Fetching WFS GetCapabilities and searching for '{keyword}'...") + resp = requests.get(WFS_URL, params={ + "SERVICE": "WFS", "VERSION": "2.0.0", "REQUEST": "GetCapabilities", + }, timeout=60) + resp.raise_for_status() + + import re + names = re.findall(r"([^<]*)", resp.text) + matches = [n for n in names if keyword.lower() in n.lower()] + print(f"Found {len(matches)} matching layer(s):") + for m in matches: + print(f" {m}") + return matches + + +def check_typename(type_name: str) -> bool: + """Tiny test query (count=1, no bbox) to confirm a typeName is valid + before running the real extraction against it.""" + params = { + "SERVICE": "WFS", "VERSION": "2.0.0", "REQUEST": "GetFeature", + "TYPENAMES": type_name, "COUNT": 1, "OUTPUTFORMAT": "application/json", + } + resp = requests.get(WFS_URL, params=params, timeout=30) + ok, error_detail = _parse_wfs_response(resp) + status = "OK" if ok else f"FAILED" + print(f" {type_name}: {status}") + if not ok: + print(f" {error_detail}") + return ok + + +def _parse_wfs_response(resp: "requests.Response") -> "tuple[bool, str]": + """ + Check whether a WFS response is a real FeatureCollection or an error + (WFS ExceptionReport, or a non-JSON/HTML error page). Returns + (is_valid, detail) -- detail is empty on success, otherwise a + human-readable explanation of what actually came back, so a failure + is never silently swallowed into "0 features". + """ + if resp.status_code != 200: + return False, f"HTTP {resp.status_code}: {resp.text[:400]}" + + try: + data = resp.json() + except ValueError: + # Not JSON at all -- almost always means an XML ExceptionReport + # or an HTML error page came back instead. + return False, f"Response was not JSON (likely a WFS ExceptionReport): {resp.text[:400]}" + + if isinstance(data, dict) and data.get("type") == "FeatureCollection": + return True, "" + + # Some WFS error responses come back as JSON but without a + # FeatureCollection shape (e.g. {"exceptionText": [...]}) + return False, f"Response was JSON but not a FeatureCollection: {str(data)[:400]}" + + +def fetch_features(type_name: str, bbox: tuple, out_path: Path, srs: str = "EPSG:4326") -> None: + """ + GetFeature request over the given bbox, paginated via startIndex + since a national layer clipped to a small bbox can still exceed the + server's per-request feature limit. + + AXIS ORDER: when BBOX specifies its CRS via the URN form + (urn:ogc:def:crs:EPSG::4326), the OGC spec requires lat,lon axis + order -- NOT the traditional lon,lat most tools use. Getting this + backwards doesn't error, it just silently matches zero real-world + features (a well-known WFS 2.0 gotcha). Rather than guess, this + tries lon,lat first and automatically retries with lat,lon swapped + if the first attempt comes back empty, so a silent axis-order bug + can't hide as "no data in this area" again. + """ + min_lon, min_lat, max_lon, max_lat = bbox + orderings = [ + ("lon,lat", f"{min_lon},{min_lat},{max_lon},{max_lat},urn:ogc:def:crs:EPSG::4326"), + ("lat,lon", f"{min_lat},{min_lon},{max_lat},{max_lon},urn:ogc:def:crs:EPSG::4326"), + ] + + all_features = [] + for label, bbox_param in orderings: + print(f" trying axis order {label}...") + all_features = _fetch_paginated(type_name, bbox_param) + if all_features: + print(f" -> {label} worked ({len(all_features)} feature(s)). " + f"Use this axis order for future queries against this endpoint.") + break + print(f" -> {label} returned 0 features.") + + import json + geojson = {"type": "FeatureCollection", "features": all_features} + out_path.write_text(json.dumps(geojson)) + if all_features: + print(f"Saved {len(all_features)} feature(s) to {out_path}") + else: + print(f"Saved an EMPTY file to {out_path} -- both axis orders returned " + f"nothing real. The typeName is valid (confirmed by --check), so this " + f"suggests the bbox itself doesn't overlap any features in this layer, " + f"or another parameter is off. Try a much larger bbox (e.g. all of " + f"France: -5,41,10,51) as a sanity check.") + + +def _fetch_paginated(type_name: str, bbox_param: str) -> list: + """One bbox ordering's worth of paginated GetFeature calls.""" + all_features = [] + start_index = 0 + page_size = 1000 + + while True: + params = { + "SERVICE": "WFS", "VERSION": "2.0.0", "REQUEST": "GetFeature", + "TYPENAMES": type_name, "BBOX": bbox_param, + "OUTPUTFORMAT": "application/json", + "COUNT": page_size, "STARTINDEX": start_index, + } + resp = requests.get(WFS_URL, params=params, timeout=60) + is_valid, error_detail = _parse_wfs_response(resp) + if not is_valid: + print(f" ERROR at startIndex={start_index}: {error_detail}") + break + data = resp.json() + features = data.get("features", []) + if not features: + break + all_features.extend(features) + if len(features) < page_size: + break + start_index += page_size + + return all_features + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download BD TOPO hydrography for Eure/Risle") + parser.add_argument("--check", action="store_true", + help="Only verify typeNames, don't download anything") + parser.add_argument("--troncon-typename", default=DEFAULT_TRONCON_TYPENAME) + parser.add_argument("--surface-typename", default=DEFAULT_SURFACE_TYPENAME) + parser.add_argument("--catchment-typename", default=DEFAULT_CATCHMENT_TYPENAME) + parser.add_argument("--skip-catchments", action="store_true", + help="Skip bassin_versant_topographique (untested typeName)") + parser.add_argument("--output-dir", type=Path, default=Path("datasets/bdtopo_hydro")) + args = parser.parse_args() + + if args.check: + print("=" * 70) + print("Testing default typeNames...") + print("=" * 70) + ok_troncon = check_typename(args.troncon_typename) + ok_surface = check_typename(args.surface_typename) + ok_catchment = args.skip_catchments or check_typename(args.catchment_typename) + if not (ok_troncon and ok_surface and ok_catchment): + print() + print("At least one default typeName failed. Searching capabilities " + "for the real name(s) instead:") + find_hydro_typenames("hydrographique") + find_hydro_typenames("bassin_versant") + print() + print("Re-run with --troncon-typename / --surface-typename / " + "--catchment-typename set to whatever the search above found, " + "then --check again to confirm.") + else: + print() + print("All typeNames OK. Re-run without --check to download the real data.") + return + + args.output_dir.mkdir(parents=True, exist_ok=True) + + print("Fetching troncon_hydrographique (river centerline geometry + attributes)...") + fetch_features(args.troncon_typename, BBOX, args.output_dir / "troncon_hydrographique.geojson") + + print() + print("Fetching surface_hydrographique (includes karst/Nature classification)...") + fetch_features(args.surface_typename, BBOX, args.output_dir / "surface_hydrographique.geojson") + + if not args.skip_catchments: + print() + print("Fetching bassin_versant_topographique (catchment polygons)...") + print("NOTE: this typeName is unverified -- if it fails, run --check first " + "to find the real name, or pass --skip-catchments to skip it.") + fetch_features(args.catchment_typename, BBOX, args.output_dir / "bassin_versant_topographique.geojson") + + print() + print("Done. Load with: BDTopoHydroLoader(data_path=...)") + print("To check for the karst-flagged reach near Grosley-sur-Risle/Ajou, use " + "loader.check_karst_near_point(lat, lon) -- that's the check that would " + "confirm or deny the bétoire location.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/download_c3s_seasonal.py b/scripts/data_acquisition/download_c3s_seasonal.py new file mode 100644 index 0000000000000000000000000000000000000000..a58914c298dcd3f9d66025fb544c76415df3b7b9 --- /dev/null +++ b/scripts/data_acquisition/download_c3s_seasonal.py @@ -0,0 +1,181 @@ +""" +Downloads real seasonal forecast data from Copernicus's C3S service -- +genuine physics-based forecasts from operational centers (ECMWF's SEAS5 +by default), not our own statistical extrapolation from ERA5 history. +Same account/library (cdsapi) already used for download_era5_full.py. + +CONFIRMED (via ECMWF/Copernicus documentation, not guessed): dataset +'seasonal-original-single-levels' for daily-resolution data, real +request parameters originating_centre/system/variable/year/month/ +leadtime_hour/area/format. GRIB is the CDS default format; netCDF +("experimental" per ECMWF's own docs) must be requested explicitly to +match this project's existing xarray-based pipeline (SAFRANLoader etc.) +rather than adding a new GRIB-reading dependency. + +NOT independently confirmed, flagged rather than assumed: + - The exact 'system' version number for ECMWF SEAS5 -- these get + revised over time by the provider. Run --check first; if it fails, + check the real current value at + https://cds.climate.copernicus.eu/datasets/seasonal-original-single-levels + - Whether a separate monthly-statistics dataset (rather than daily + with an hourly leadtime) is the better fit here -- --frequency lets + you try both; daily-with-leadtime_hour is what's directly confirmed + in the docs, so it's the default. + +AREA / AXIS ORDER: the CDS 'area' parameter uses [North, West, South, +East] order -- confirmed different from this project's own +(min_lon, min_lat, max_lon, max_lat) bbox convention used everywhere +else (download_bdtopo_hydro.py, download_bdcavites.py). Converted +explicitly in build_area_param below rather than passed through +unchanged -- getting an axis order wrong has silently broken a request +in this project before (BD TOPO's WFS), so this is checked directly, +not assumed compatible. + +Two request types, genuinely different, both needed: + - hindcasts (1993-2016): historical reforecasts, needed to bias- + correct/calibrate the real-time forecast against local climatology + the same way climate_climatology.py already does for ERA5. + - real-time forecasts (2017+): the actual current forecast to use at + inference time. + +Usage: + python -m scripts.download_c3s_seasonal --check + python -m scripts.download_c3s_seasonal --mode hindcast --month 1 + python -m scripts.download_c3s_seasonal --mode forecast --year 2026 --month 8 +""" +import argparse +from pathlib import Path + +# Matches download_bdtopo_hydro.py's widened bbox, same study area. +BBOX = (-0.1, 47.7, 2.1, 49.9) # (min_lon, min_lat, max_lon, max_lat) + +DATASET = "seasonal-original-single-levels" +DEFAULT_ORIGINATING_CENTRE = "ecmwf" +DEFAULT_SYSTEM = "51" # SEAS5 -- NOT independently confirmed, see module docstring + +# ERA5-style variable names, matching this project's existing SAFRANLoader +# VAR_MAP naming convention where the CDS names line up (t2m, tp) -- +# NOT independently confirmed identical for the seasonal dataset +# specifically; run --check before trusting this list. +DEFAULT_VARIABLES = ["2m_temperature", "total_precipitation"] + + +def build_area_param(bbox: tuple) -> list: + """ + CDS 'area' is [North, West, South, East] -- confirmed different + from this project's own (min_lon, min_lat, max_lon, max_lat) + convention. Converts explicitly rather than passing bbox through + unchanged. + """ + min_lon, min_lat, max_lon, max_lat = bbox + return [max_lat, min_lon, min_lat, max_lon] # [North, West, South, East] + + +def build_hindcast_request(month: int, variables: list, system: str, originating_centre: str) -> dict: + """ + One request per initialization MONTH, covering all hindcast years + at once -- ECMWF's own efficiency guidance (confirmed in their + documentation) is that this is much more efficient than looping + per-year, since a single month across many years reuses the same + underlying data partition. + """ + return { + "originating_centre": originating_centre, + "system": system, + "variable": variables, + "year": [str(y) for y in range(1993, 2017)], + "month": [f"{month:02d}"], + "day": ["01"], + "leadtime_hour": [str(h) for h in range(24, 24 * 186, 24)], # daily steps out to ~6 months + "area": build_area_param(BBOX), + "format": "netcdf", + } + + +def build_forecast_request(year: int, month: int, variables: list, system: str, originating_centre: str) -> dict: + """One real-time forecast initialization (a specific year+month), same variables/area/format as hindcasts.""" + return { + "originating_centre": originating_centre, + "system": system, + "variable": variables, + "year": [str(year)], + "month": [f"{month:02d}"], + "day": ["01"], + "leadtime_hour": [str(h) for h in range(24, 24 * 186, 24)], + "area": build_area_param(BBOX), + "format": "netcdf", + } + + +def check_access(originating_centre: str, system: str) -> bool: + """Small real request (single variable, single hindcast month, short + leadtime range) to validate credentials/dataset/system/variable + names actually work before committing to a full download.""" + import cdsapi + + client = cdsapi.Client() + request = { + "originating_centre": originating_centre, + "system": system, + "variable": ["2m_temperature"], + "year": ["2015"], + "month": ["01"], + "day": ["01"], + "leadtime_hour": ["24", "48"], + "area": build_area_param(BBOX), + "format": "netcdf", + } + print(f"Testing {DATASET} with originating_centre={originating_centre!r}, system={system!r}...") + try: + client.retrieve(DATASET, request, "/tmp/c3s_check.nc") + print("OK -- request succeeded, real data downloaded to /tmp/c3s_check.nc") + return True + except Exception as e: + print(f"FAILED: {e}") + print(f"Check the real current dataset/parameter values at " + f"https://cds.climate.copernicus.eu/datasets/{DATASET}") + return False + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download C3S seasonal forecast data") + parser.add_argument("--check", action="store_true") + parser.add_argument("--mode", choices=["hindcast", "forecast"], default="hindcast") + parser.add_argument("--month", type=int, required=False, help="Initialization month (1-12)") + parser.add_argument("--year", type=int, required=False, help="Forecast year (--mode forecast only)") + parser.add_argument("--originating-centre", type=str, default=DEFAULT_ORIGINATING_CENTRE) + parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM) + parser.add_argument("--variables", type=str, nargs="+", default=DEFAULT_VARIABLES) + parser.add_argument("--output-dir", type=Path, default=Path("datasets/c3s_seasonal")) + args = parser.parse_args() + + if args.check: + check_access(args.originating_centre, args.system) + return + + if args.month is None: + print("--month is required (1-12) unless using --check") + return + + import cdsapi + client = cdsapi.Client() + args.output_dir.mkdir(parents=True, exist_ok=True) + + if args.mode == "hindcast": + request = build_hindcast_request(args.month, args.variables, args.system, args.originating_centre) + out_path = args.output_dir / f"hindcast_month{args.month:02d}.nc" + else: + if args.year is None: + print("--year is required for --mode forecast") + return + request = build_forecast_request(args.year, args.month, args.variables, args.system, args.originating_centre) + out_path = args.output_dir / f"forecast_{args.year}_{args.month:02d}.nc" + + print(f"Requesting {DATASET} ({args.mode}, month={args.month})...") + print("This can take a while -- CDS queues large seasonal forecast requests.") + client.retrieve(DATASET, request, str(out_path)) + print(f"Saved to {out_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/download_catchment.py b/scripts/data_acquisition/download_catchment.py new file mode 100644 index 0000000000000000000000000000000000000000..b6c50ad958a7562d4ec588df40a7cf34081f40b8 --- /dev/null +++ b/scripts/data_acquisition/download_catchment.py @@ -0,0 +1,114 @@ +""" +Download catchment (drainage basin) area for all stations from Hub'Eau's +own hydrometrie referentiel — the same API download_hubeau.py already +uses, just a different endpoint. + +Catchment area is NOT on the station referentiel directly; it lives on +the SITE referentiel (`surface_bv`, "Superficie du BV en km2"). A site +can have multiple stations, so this does two passes: + 1. referentiel/stations -> get each station's code_site + 2. referentiel/sites -> get surface_bv for those code_site values + 3. join back onto station_code + +This gives real drainage-area data usable for area-ratio discharge +scaling between gauges on the same river (Q_ungauged ~= Q_gauged * +(area_ungauged / area_gauged)) — a physically grounded alternative to +naive geographic proximity interpolation. +""" +import requests +import pandas as pd +from pathlib import Path + +BASE_URL = "https://hubeau.eaufrance.fr/api/v2/hydrometrie" +OUTPUT_DIR = Path("datasets") +OUTPUT_DIR.mkdir(exist_ok=True) + +stations_df = pd.read_csv("datasets/station_list.csv") +station_codes = stations_df["station_code"].tolist() + +print(f"Fetching catchment area for {len(station_codes)} stations...") +print() + + +def fetch_all(endpoint: str, params: dict) -> list: + """Fetch every page from a Hub'Eau referentiel endpoint (cursor pagination).""" + all_data = [] + cursor = "" + while True: + query = dict(params, cursor=cursor, size=1000, format="json") + response = requests.get(f"{BASE_URL}/{endpoint}", params=query, timeout=30) + if response.status_code not in (200, 206): + print(f" Error {response.status_code}: {response.text[:200]}") + break + data = response.json() + results = data.get("data", []) + if not results: + break + all_data.extend(results) + next_url = data.get("next") + if not next_url: + break + if "cursor=" in next_url: + cursor = next_url.split("cursor=")[1].split("&")[0] + else: + break + return all_data + + +# --- Step 1: station -> code_site ----------------------------------------- +print("=" * 80) +print("STEP 1: Looking up code_site for each station") +print("=" * 80) + +station_to_site = {} +# code_station accepts up to 100 comma-separated codes per request +for i in range(0, len(station_codes), 100): + batch = station_codes[i:i + 100] + results = fetch_all("referentiel/stations", {"code_station": ",".join(batch)}) + for r in results: + station_to_site[r["code_station"]] = r.get("code_site") + print(f" {r['code_station']} -> site {r.get('code_site')}") + +missing = set(station_codes) - set(station_to_site) +if missing: + print(f"\n Warning: {len(missing)} station(s) not found in referentiel/stations: {missing}") + +# --- Step 2: code_site -> surface_bv --------------------------------------- +print() +print("=" * 80) +print("STEP 2: Fetching surface_bv (catchment area) for each site") +print("=" * 80) + +site_codes = sorted(set(s for s in station_to_site.values() if s)) +site_to_area = {} +for i in range(0, len(site_codes), 100): + batch = site_codes[i:i + 100] + results = fetch_all("referentiel/sites", {"code_site": ",".join(batch)}) + for r in results: + area = r.get("surface_bv") + site_to_area[r["code_site"]] = area + print(f" site {r['code_site']} ({r.get('libelle_site', '?')}): " + f"{area if area is not None else 'no data'} km²") + +# --- Step 3: join back onto station_code ----------------------------------- +rows = [] +for code in station_codes: + site = station_to_site.get(code) + area = site_to_area.get(site) if site else None + rows.append({"station_code": code, "code_site": site, "catchment_area_km2": area}) + +result_df = pd.DataFrame(rows) +output_file = OUTPUT_DIR / "catchment_area.csv" +result_df.to_csv(output_file, index=False) + +print() +print("=" * 80) +print("DONE") +print("=" * 80) +n_ok = result_df["catchment_area_km2"].notna().sum() +print(f"Catchment area found for {n_ok}/{len(result_df)} stations") +print(f"Saved to: {output_file}") +if n_ok < len(result_df): + missing_codes = result_df[result_df["catchment_area_km2"].isna()]["station_code"].tolist() + print(f"\nNo area found for: {missing_codes}") + print("(Hub'Eau doesn't publish surface_bv for every site — some are genuinely blank.)") \ No newline at end of file diff --git a/scripts/data_acquisition/download_elevation.py b/scripts/data_acquisition/download_elevation.py new file mode 100644 index 0000000000000000000000000000000000000000..c82e7482e390c0dbbb984797a129ff51b064b106 --- /dev/null +++ b/scripts/data_acquisition/download_elevation.py @@ -0,0 +1,67 @@ +"""Download elevation data for all 27 stations using Open Topo Data API.""" +import requests +import pandas as pd +import time +from pathlib import Path + +# Load stations +stations = pd.read_csv('datasets/station_list.csv') + +print(f"Downloading elevation for {len(stations)} stations...") + +elevations = [] + +for idx, row in stations.iterrows(): + station_code = row['station_code'] + lat = row['lat'] + lon = row['lon'] + + # Query Open Topo Data API + url = f"https://api.opentopodata.org/v1/eudem25m?locations={lat},{lon}" + + try: + response = requests.get(url, timeout=10) + if response.status_code == 200: + data = response.json() + elevation = data['results'][0]['elevation'] + elevations.append({ + 'station_code': station_code, + 'station_name': row['station_name'], + 'latitude': lat, + 'longitude': lon, + 'elevation_m': elevation, + 'data_source': 'EU-DEM 25m (Open Topo Data)' + }) + print(f"✓ {station_code}: {elevation:.1f} m") + else: + print(f"✗ {station_code}: API error {response.status_code}") + elevations.append({ + 'station_code': station_code, + 'station_name': row['station_name'], + 'latitude': lat, + 'longitude': lon, + 'elevation_m': None, + 'data_source': 'Failed' + }) + except Exception as e: + print(f"✗ {station_code}: {e}") + elevations.append({ + 'station_code': station_code, + 'station_name': row['station_name'], + 'latitude': lat, + 'longitude': lon, + 'elevation_m': None, + 'data_source': 'Failed' + }) + + # Rate limiting + time.sleep(3) + +# Save +df_elevation = pd.DataFrame(elevations) +output_file = Path('datasets/station_elevations.csv') +df_elevation.to_csv(output_file, index=False) + +print(f"\n✓ Saved to: {output_file}") +print(f"✓ Success: {df_elevation['elevation_m'].notna().sum()}/{len(df_elevation)} stations") +print(f"\nElevation range: {df_elevation['elevation_m'].min():.1f} - {df_elevation['elevation_m'].max():.1f} m") diff --git a/scripts/data_acquisition/download_era5_full.py b/scripts/data_acquisition/download_era5_full.py new file mode 100644 index 0000000000000000000000000000000000000000..edbde96965581f84fdc773f31caf6f1bac02bf02 --- /dev/null +++ b/scripts/data_acquisition/download_era5_full.py @@ -0,0 +1,180 @@ +""" +Download full ERA5 dataset (1960-2026) for all 27 stations. +Downloads year by year with separate requests for instantaneous vs accumulated variables. + +NOTE: ERA5 separates instantaneous (temperature, wind) from accumulated +(precipitation, evaporation, radiation) variables. They must be requested separately. +""" +import cdsapi +import pandas as pd +import xarray as xr +import zipfile +import shutil +from pathlib import Path + +# Load station coordinates +stations = pd.read_csv('datasets/station_list.csv') +lat_min = stations['lat'].min() - 0.1 +lat_max = stations['lat'].max() + 0.1 +lon_min = stations['lon'].min() - 0.1 +lon_max = stations['lon'].max() + 0.1 + +print("="*80) +print("ERA5 FULL DOWNLOAD (1960-2026)") +print("="*80) +print(f"Study area: [{lat_min:.2f}, {lon_min:.2f}] to [{lat_max:.2f}, {lon_max:.2f}]") +print(f"Years: 1960-2026 (67 years)") +print(f"Output: datasets/safran/") +print("="*80) + +# Split variables by type (instantaneous vs accumulated) +instantaneous_vars = [ + '2m_temperature', # T_Q + '10m_u_component_of_wind', # FF_Q + '10m_v_component_of_wind', # FF_Q +] + +accumulated_vars = [ + 'total_precipitation', # PRELIQ_Q + 'potential_evaporation', # ETP_Q + 'surface_solar_radiation_downwards', # DLI_Q / SSI_Q + 'snowfall', # PRENEI_Q + 'runoff', # RUNC_Q +] + +def extract_netcdf(zip_file): + """Extract NetCDF from ZIP archive.""" + if zipfile.is_zipfile(zip_file): + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + # Find all .nc files in the zip + nc_files = [f for f in zip_ref.namelist() if f.endswith('.nc')] + extracted_files = [] + for nc_file in nc_files: + temp_file = zip_file.parent / f'temp_{nc_file}' + with zip_ref.open(nc_file) as source: + with open(temp_file, 'wb') as target: + shutil.copyfileobj(source, target) + extracted_files.append(temp_file) + zip_file.unlink() + return extracted_files + return [zip_file] + +def merge_datasets(instant_file, accum_file, output_file): + """Merge instantaneous and accumulated variable datasets.""" + ds_instant = xr.open_dataset(instant_file) + ds_accum = xr.open_dataset(accum_file) + + # Merge datasets + ds_merged = xr.merge([ds_instant, ds_accum]) + ds_merged.to_netcdf(output_file) + + ds_instant.close() + ds_accum.close() + instant_file.unlink() + accum_file.unlink() + +c = cdsapi.Client() +output_dir = Path('datasets/safran') +output_dir.mkdir(exist_ok=True) + +# Download year by year (1960-2026) +years = list(range(1960, 2027)) # 1960 to 2026 inclusive +failed_years = [] + +for year in years: + output_file = output_dir / f'era5_{year}.nc' + + # Skip if already exists and contains all 8 variables + if output_file.exists(): + try: + ds = xr.open_dataset(output_file) + if len(ds.data_vars) >= 8: + print(f"✓ {year} - Already downloaded with all variables, skipping") + ds.close() + continue + else: + print(f"⚠ {year} - Incomplete ({len(ds.data_vars)} vars), re-downloading") + ds.close() + output_file.unlink() + except: + output_file.unlink() + + print(f"\nDownloading {year}...") + + instant_file = output_dir / f'era5_{year}_instant.nc' + accum_file = output_dir / f'era5_{year}_accum.nc' + + try: + # Download instantaneous variables + print(f" → Instantaneous variables...") + c.retrieve( + 'reanalysis-era5-single-levels', + { + 'product_type': 'reanalysis', + 'format': 'netcdf', + 'variable': instantaneous_vars, + 'year': str(year), + 'month': [f'{m:02d}' for m in range(1, 13)], + 'day': [f'{d:02d}' for d in range(1, 32)], + 'time': '12:00', + 'area': [lat_max, lon_min, lat_min, lon_max], + }, + str(instant_file) + ) + + # Extract if ZIP + extracted = extract_netcdf(instant_file) + if len(extracted) > 0: + instant_file = extracted[0] + + # Download accumulated variables + print(f" → Accumulated variables...") + c.retrieve( + 'reanalysis-era5-single-levels', + { + 'product_type': 'reanalysis', + 'format': 'netcdf', + 'variable': accumulated_vars, + 'year': str(year), + 'month': [f'{m:02d}' for m in range(1, 13)], + 'day': [f'{d:02d}' for d in range(1, 32)], + 'time': '12:00', + 'area': [lat_max, lon_min, lat_min, lon_max], + }, + str(accum_file) + ) + + # Extract if ZIP + extracted = extract_netcdf(accum_file) + if len(extracted) > 0: + accum_file = extracted[0] + + # Merge both datasets + print(f" → Merging datasets...") + merge_datasets(instant_file, accum_file, output_file) + + size_kb = output_file.stat().st_size / 1024 + print(f"✓ {year} - Downloaded ({size_kb:.1f} KB)") + + except Exception as e: + print(f"✗ {year} - Failed: {e}") + failed_years.append(year) + # Cleanup partial files + for f in [instant_file, accum_file]: + if f.exists(): + f.unlink() + continue + +print("\n" + "="*80) +print("DOWNLOAD COMPLETE") +print("="*80) +print(f"Successful: {len(years) - len(failed_years)}/{len(years)} years") + +if failed_years: + print(f"\nFailed years: {failed_years}") + print("You can re-run this script to retry failed downloads") +else: + print("\n✓ All years downloaded successfully!") + +print(f"\nFiles saved to: {output_dir}") +print("Total files:", len(list(output_dir.glob('era5_*.nc')))) diff --git a/scripts/data_acquisition/download_era5_sample.py b/scripts/data_acquisition/download_era5_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..61279a260ab3e1448f21cb824faf8fee88c19939 --- /dev/null +++ b/scripts/data_acquisition/download_era5_sample.py @@ -0,0 +1,44 @@ +"""Download ERA5 sample (Jan 2020) to test setup.""" +import cdsapi +import pandas as pd +from pathlib import Path + +# Load station coordinates for bounding box +stations = pd.read_csv('datasets/station_list.csv') +lat_min = stations['lat'].min() - 0.1 +lat_max = stations['lat'].max() + 0.1 +lon_min = stations['lon'].min() - 0.1 +lon_max = stations['lon'].max() + 0.1 + +print(f"Study area: [{lat_min:.2f}, {lon_min:.2f}] to [{lat_max:.2f}, {lon_max:.2f}]") +print("Downloading ERA5 sample: January 2020...") + +c = cdsapi.Client() + +output_file = Path('datasets/safran/era5_2020_jan.nc') + +# Download sample month +c.retrieve( + 'reanalysis-era5-single-levels', + { + 'product_type': 'reanalysis', + 'format': 'netcdf', + 'variable': [ + '2m_temperature', + 'total_precipitation', + 'potential_evaporation', + 'surface_solar_radiation_downwards', + '10m_u_component_of_wind', + '10m_v_component_of_wind', + ], + 'year': '2020', + 'month': '01', + 'day': [f'{d:02d}' for d in range(1, 32)], + 'time': '12:00', + 'area': [lat_max, lon_min, lat_min, lon_max], + }, + str(output_file) +) + +print(f"✓ Downloaded: {output_file}") +print(f"✓ Size: {output_file.stat().st_size / 1024 / 1024:.1f} MB") diff --git a/scripts/data_acquisition/download_hubeau.py b/scripts/data_acquisition/download_hubeau.py new file mode 100644 index 0000000000000000000000000000000000000000..586eedb5428ab7e8887c493d1dd123bcf7806730 --- /dev/null +++ b/scripts/data_acquisition/download_hubeau.py @@ -0,0 +1,162 @@ +""" +Download hydrometric data from Hub'Eau API v2. +Downloads QmJ (daily discharge), HIJ (daily water level) for all 27 stations. +""" +import requests +import pandas as pd +from pathlib import Path +from tqdm import tqdm +import time + +# Configuration +OUTPUT_DIR = Path("datasets/hydrometric") +OUTPUT_DIR.mkdir(exist_ok=True) + +BASE_URL = "https://hubeau.eaufrance.fr/api/v2/hydrometrie" + +# Load stations +stations_df = pd.read_csv("datasets/station_list.csv") +station_codes = stations_df["station_code"].tolist() + +print(f"Downloading hydrometric data for {len(station_codes)} stations...") +print(f"Output: {OUTPUT_DIR}") +print() + +def download_observations(station_code, grandeur_hydro="Q", max_results=100000): + """ + Download observations for a station. + + Args: + station_code: Station code (e.g., H602021010) + grandeur_hydro: H (water level) or Q (discharge) + max_results: Maximum results to fetch + + Returns: + DataFrame with observations + """ + url = f"{BASE_URL}/obs_elab" + + all_data = [] + cursor = "" + page = 1 + + while len(all_data) < max_results: + params = { + "code_entite": station_code, + "grandeur_hydro": grandeur_hydro, + "size": 20000, # Max per request + "format": "json", + "cursor": cursor + } + + try: + response = requests.get(url, params=params, timeout=30) + + if response.status_code == 200: + data = response.json() + results = data.get("data", []) + + if not results: + break + + all_data.extend(results) + + # Check for next page + next_url = data.get("next") + if not next_url or len(results) < 20000: + break + + # Extract cursor from next URL + if "cursor=" in next_url: + cursor = next_url.split("cursor=")[1].split("&")[0] + else: + break + + page += 1 + time.sleep(0.1) # Rate limiting + + else: + print(f" Error {response.status_code}") + break + + except Exception as e: + print(f" Exception: {e}") + break + + if all_data: + return pd.DataFrame(all_data) + else: + return pd.DataFrame() + + +# Download discharge (Q) for all stations +print("="*80) +print("DOWNLOADING DISCHARGE (Q)") +print("="*80) + +discharge_data = [] + +for station_code in tqdm(station_codes, desc="Discharge"): + df = download_observations(station_code, grandeur_hydro="Q") + + if not df.empty: + df["station_code"] = station_code + discharge_data.append(df) + tqdm.write(f"✓ {station_code}: {len(df)} observations") + else: + tqdm.write(f"✗ {station_code}: No data") + + time.sleep(0.2) + +if discharge_data: + discharge_df = pd.concat(discharge_data, ignore_index=True) + + # Save + discharge_df.to_csv(OUTPUT_DIR / "discharge_observations.csv", index=False) + discharge_df.to_json(OUTPUT_DIR / "discharge_observations.json", orient="records", indent=2) + + print(f"\n✓ Discharge: {len(discharge_df)} total observations") + print(f" Stations: {discharge_df['station_code'].nunique()}") + print(f" Date range: {discharge_df['date_obs_elab'].min()} to {discharge_df['date_obs_elab'].max()}") + + +# Download water level (H) for all stations +print("\n" + "="*80) +print("DOWNLOADING WATER LEVEL (H)") +print("="*80) + +waterlevel_data = [] + +for station_code in tqdm(station_codes, desc="Water Level"): + df = download_observations(station_code, grandeur_hydro="H") + + if not df.empty: + df["station_code"] = station_code + waterlevel_data.append(df) + tqdm.write(f"✓ {station_code}: {len(df)} observations") + else: + tqdm.write(f"✗ {station_code}: No data") + + time.sleep(0.2) + +if waterlevel_data: + waterlevel_df = pd.concat(waterlevel_data, ignore_index=True) + + # Save + waterlevel_df.to_csv(OUTPUT_DIR / "waterlevel_observations.csv", index=False) + waterlevel_df.to_json(OUTPUT_DIR / "waterlevel_observations.json", orient="records", indent=2) + + print(f"\n✓ Water level: {len(waterlevel_df)} total observations") + print(f" Stations: {waterlevel_df['station_code'].nunique()}") + print(f" Date range: {waterlevel_df['date_obs_elab'].min()} to {waterlevel_df['date_obs_elab'].max()}") + + +# Summary +print("\n" + "="*80) +print("DOWNLOAD COMPLETE") +print("="*80) +print(f"Files saved to: {OUTPUT_DIR}") +print(f"\nFiles created:") +for f in OUTPUT_DIR.glob("*"): + size_mb = f.stat().st_size / 1024 / 1024 + print(f" - {f.name} ({size_mb:.1f} MB)") diff --git a/scripts/data_acquisition/extract_era5.py b/scripts/data_acquisition/extract_era5.py new file mode 100644 index 0000000000000000000000000000000000000000..293c00dab6772851e5f89c3d326a08a77d0c6860 --- /dev/null +++ b/scripts/data_acquisition/extract_era5.py @@ -0,0 +1,57 @@ +""" +Extract NetCDF files from ERA5 ZIP archives. +CDS API downloads NetCDF data as compressed ZIP files. +""" +import zipfile +from pathlib import Path +import shutil + +safran_dir = Path('datasets/safran') + +print("=" * 80) +print("EXTRACTING ERA5 NETCDF FILES FROM ZIP ARCHIVES") +print("=" * 80) + +# Find all .nc files that are actually ZIP archives +nc_files = list(safran_dir.glob('era5_*.nc')) +extracted = 0 +skipped = 0 + +for nc_file in sorted(nc_files): + # Check if it's a ZIP file + try: + with zipfile.ZipFile(nc_file, 'r') as zip_ref: + # It's a ZIP file - extract it + zip_contents = zip_ref.namelist() + + # Find the actual NetCDF file inside (usually data.nc or similar) + actual_nc = [f for f in zip_contents if f.endswith('.nc')][0] + + # Extract to temporary location + temp_file = safran_dir / f'temp_{nc_file.name}' + with zip_ref.open(actual_nc) as source: + with open(temp_file, 'wb') as target: + shutil.copyfileobj(source, target) + + # Replace the ZIP with the extracted NetCDF + nc_file.unlink() + temp_file.rename(nc_file) + + size_kb = nc_file.stat().st_size / 1024 + print(f"✓ Extracted {nc_file.name} ({size_kb:.1f} KB)") + extracted += 1 + + except zipfile.BadZipFile: + # Not a ZIP file - it's already a proper NetCDF, skip + size_kb = nc_file.stat().st_size / 1024 + print(f" {nc_file.name} - Already extracted ({size_kb:.1f} KB)") + skipped += 1 + except Exception as e: + print(f"✗ Failed to extract {nc_file.name}: {e}") + +print("\n" + "=" * 80) +print("EXTRACTION COMPLETE") +print("=" * 80) +print(f"Extracted: {extracted} files") +print(f"Already extracted: {skipped} files") +print(f"Total NetCDF files: {len(nc_files)}") diff --git a/scripts/data_acquisition/fetch_ecmwf_forecast.py b/scripts/data_acquisition/fetch_ecmwf_forecast.py new file mode 100644 index 0000000000000000000000000000000000000000..572d031fe4d2bca0d2baab337a2d43edf077aee6 --- /dev/null +++ b/scripts/data_acquisition/fetch_ecmwf_forecast.py @@ -0,0 +1,152 @@ +""" +Fetches real, physics-based short-to-medium-range forecast data from +Open-Meteo's dedicated ECMWF endpoint -- built specifically for genuine +flood early-warning lead time, which (per the recent design discussion) +structurally requires a real precipitation forecast, not just discharge +history extrapolated forward. + +CONFIRMED real API details (not guessed): base endpoint +https://api.open-meteo.com/v1/ecmwf, HTTP GET, no authentication, CC BY +4.0 licence, up to 10 real forecast days. Also confirmed to exist (not +yet used here): an archived-individual-model-run-by-init-time feature +explicitly described by Open-Meteo as built for bias-correction +pipelines without look-ahead leakage -- exactly what a later step would +need to calibrate this forecast against our own real climatology (see +climate_climatology.py). The exact endpoint for that archive wasn't +directly confirmed in this pass -- flagged for a follow-up check, not +guessed at here. + +Usage: + python -m scripts.fetch_ecmwf_forecast --check + python -m scripts.fetch_ecmwf_forecast --stations-path datasets/station_elevations.csv +""" +import argparse +from pathlib import Path + +import pandas as pd +import requests + +BASE_URL = "https://api.open-meteo.com/v1/ecmwf" + +# Daily aggregates, matching this project's existing daily-resolution +# pipeline convention (ERA5 climatology etc.) rather than pulling hourly +# data that would need its own separate aggregation step. +DAILY_VARIABLES = [ + "precipitation_sum", "temperature_2m_max", "temperature_2m_min", + "wind_speed_10m_max", "shortwave_radiation_sum", "et0_fao_evapotranspiration", +] + + +def check_access() -> bool: + """Single real request against a real coordinate in the study area + (matches H602021010's real location) to confirm the endpoint and + variable names actually work before looping over every station.""" + params = { + "latitude": 48.748705218, "longitude": 0.58156195, + "daily": ",".join(DAILY_VARIABLES), "forecast_days": 5, + "timezone": "Europe/Paris", + } + print(f"Testing {BASE_URL} with a real study-area coordinate...") + try: + resp = requests.get(BASE_URL, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + except Exception as e: + print(f"FAILED: {e}") + return False + + if "daily" not in data: + print(f"FAILED: response has no 'daily' key. Full response: {data}") + return False + + print("OK -- real forecast returned:") + for var in DAILY_VARIABLES: + if var in data["daily"]: + print(f" {var}: {data['daily'][var][:3]}... ({len(data['daily'][var])} real day(s))") + else: + print(f" WARNING: '{var}' not found in response -- variable name may be wrong. " + f"Real keys returned: {list(data['daily'].keys())}") + return True + + +def fetch_forecast(lat: float, lon: float, forecast_days: int = 10, max_retries: int = 3) -> pd.DataFrame: + """ + Retries on read timeout -- confirmed as a real, not hypothetical, + intermittent failure mode against the live API: 2 of 27 real + stations hit a 30s read timeout on an otherwise-clean run, while + the other 25 succeeded immediately. A short exponential backoff + handles this automatically rather than requiring a manual re-run + for whichever stations happen to be unlucky that run. + """ + import time + + params = { + "latitude": lat, "longitude": lon, + "daily": ",".join(DAILY_VARIABLES), "forecast_days": forecast_days, + "timezone": "Europe/Paris", + } + last_error = None + for attempt in range(max_retries): + try: + resp = requests.get(BASE_URL, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + break + except requests.exceptions.RequestException as e: + last_error = e + if attempt < max_retries - 1: + wait = 2 ** attempt # 1s, 2s, 4s + print(f" attempt {attempt + 1}/{max_retries} failed ({e}), retrying in {wait}s...") + time.sleep(wait) + else: + raise last_error + + if "daily" not in data: + raise ValueError(f"No 'daily' data in response for ({lat}, {lon}): {data}") + + df = pd.DataFrame(data["daily"]) + df["latitude"] = lat + df["longitude"] = lon + return df + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch real ECMWF forecast data from Open-Meteo") + parser.add_argument("--check", action="store_true") + parser.add_argument("--stations-path", type=Path, default=Path("datasets/station_elevations.csv")) + parser.add_argument("--forecast-days", type=int, default=10) + parser.add_argument("--output", type=Path, default=Path("datasets/ecmwf_forecast.csv")) + args = parser.parse_args() + + if args.check: + check_access() + return + + stations = pd.read_csv(args.stations_path) + lat_col = "latitude" if "latitude" in stations.columns else "lat" + lon_col = "longitude" if "longitude" in stations.columns else "lon" + + all_dfs = [] + for _, row in stations.iterrows(): + code = row.get("station_code", f"{row[lat_col]:.3f}_{row[lon_col]:.3f}") + print(f"Fetching real forecast for {code}...") + try: + df = fetch_forecast(row[lat_col], row[lon_col], args.forecast_days) + df["station_code"] = code + all_dfs.append(df) + except Exception as e: + print(f" FAILED for {code}: {e}") + + if not all_dfs: + print("No real forecast data fetched at all -- check connectivity, or run --check first.") + return + + combined = pd.concat(all_dfs, ignore_index=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + combined.to_csv(args.output, index=False) + print(f"\nSaved real forecast data for {combined['station_code'].nunique()} station(s), " + f"{args.forecast_days} real day(s) each, to {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/fetch_idpr_brgm.py b/scripts/data_acquisition/fetch_idpr_brgm.py new file mode 100644 index 0000000000000000000000000000000000000000..593d75316fc6c7ba0c377fe945198e7a0de9d37f --- /dev/null +++ b/scripts/data_acquisition/fetch_idpr_brgm.py @@ -0,0 +1,144 @@ +""" +Fetches fresh IDPR (Indice de Développement et de Persistance des +Réseaux) values directly from BRGM's live geoservice, replacing/ +supplementing the manually-uploaded idpr.csv. + +CONFIRMED (fetched GetCapabilities directly): endpoint +http://geoservices.brgm.fr/geologie, WMS 1.3.0, live and responding. +IDPR is raster/grid data at five resolutions (IDPR_50M down to +IDPR_5000M) -- not a vector feature type, so WFS GetFeature (the +pattern used everywhere else in this project) doesn't apply. The +correct WMS operation for extracting a value at a specific point from a +raster layer is GetFeatureInfo: request a small map image centered on +the point, then ask "what's the value at this pixel." + +This queries ONE point at a time (GetFeatureInfo has no bulk/vectorized +equivalent the way BD TOPO's WFS BBOX queries did) -- scoped to the 27 +real gauge stations, matching what "redownload IDPR" means (fresh +values at known station points), not the full ~4,500-node reach graph +(which would mean thousands of individual HTTP requests -- a different, +much larger undertaking than this). + +Usage: + python -m scripts.fetch_idpr_brgm --check + python -m scripts.fetch_idpr_brgm --stations datasets/station_elevations.csv +""" +import argparse +import re +from pathlib import Path + +import pandas as pd +import requests + +WMS_URL = "http://geoservices.brgm.fr/geologie" +LAYER = "IDPR_50M" # highest resolution; use --layer to try a coarser one if this fails + + +def get_feature_info_at_point(lat: float, lon: float, layer: str = LAYER, + half_extent_deg: float = 0.001) -> "float | None": + """ + Query IDPR value at a single point via WMS GetFeatureInfo: request a + tiny 3x3 pixel map centered on (lat, lon), then ask for the value at + the center pixel. + """ + minx, miny = lon - half_extent_deg, lat - half_extent_deg + maxx, maxy = lon + half_extent_deg, lat + half_extent_deg + params = { + "SERVICE": "WMS", "VERSION": "1.1.1", "REQUEST": "GetFeatureInfo", + "LAYERS": layer, "QUERY_LAYERS": layer, "STYLES": "", + "BBOX": f"{minx},{miny},{maxx},{maxy}", "SRS": "EPSG:4326", + "WIDTH": 3, "HEIGHT": 3, "X": 1, "Y": 1, + "INFO_FORMAT": "text/plain", + } + try: + resp = requests.get(WMS_URL, params=params, timeout=30) + except requests.RequestException as e: + print(f" request failed: {e}") + return None + + if resp.status_code != 200: + print(f" HTTP {resp.status_code}") + return None + + return parse_getfeatureinfo_value(resp.text) + + +def parse_getfeatureinfo_value(text: str) -> "float | None": + """ + MapServer's text/plain GetFeatureInfo output is typically a small + block like: + Layer 'IDPR_50M' + Feature 0 + value = '885' + Parsing defensively: find any line with 'value' (case-insensitive) + and extract the first number on it, rather than assuming an exact + format -- MapServer's plain-text output format has enough real + variation across deployments that a strict parser is more likely to + silently return nothing than a permissive one is to return a wrong + number. + """ + for line in text.splitlines(): + if "value" in line.lower(): + match = re.search(r"[-+]?\d*\.?\d+", line) + if match: + return float(match.group()) + return None + + +def check_service() -> bool: + # A real point known to be in mainland France, well within IDPR coverage + test_lat, test_lon = 49.03, 0.79 + print(f"Testing {LAYER} at ({test_lat}, {test_lon})...") + value = get_feature_info_at_point(test_lat, test_lon) + if value is not None: + print(f" OK: value = {value}") + return True + print(" FAILED: no value returned. Try --layer IDPR_100M or IDPR_500M " + "(coarser resolutions), or check GetCapabilities directly:") + print(f" {WMS_URL}?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetCapabilities") + return False + + +def fetch_for_stations(stations_path: Path, layer: str, output_path: Path) -> None: + stations = pd.read_csv(stations_path) + lat_col = "latitude" if "latitude" in stations.columns else "lat" + lon_col = "longitude" if "longitude" in stations.columns else "lon" + + rows = [] + for _, row in stations.iterrows(): + code = row["station_code"] + value = get_feature_info_at_point(row[lat_col], row[lon_col], layer=layer) + print(f" {code}: {value}") + rows.append({"station_code": code, "idpr_value_brgm_live": value, + "latitude": row[lat_col], "longitude": row[lon_col]}) + + out = pd.DataFrame(rows) + n_ok = out["idpr_value_brgm_live"].notna().sum() + print() + print(f"{n_ok}/{len(out)} stations got a real value") + out.to_csv(output_path, index=False) + print(f"Saved to {output_path}") + if n_ok < len(out): + print("Some stations got no value -- this could mean they're just outside " + "IDPR's coverage, or the layer/resolution needs adjusting. Compare " + "against the existing idpr.csv for those specific stations before " + "assuming the live fetch is wrong.") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch fresh IDPR values from BRGM's live geoservice") + parser.add_argument("--check", action="store_true") + parser.add_argument("--stations", type=Path, default=Path("datasets/station_elevations.csv")) + parser.add_argument("--layer", type=str, default=LAYER) + parser.add_argument("--output", type=Path, default=Path("datasets/idpr_brgm_live.csv")) + args = parser.parse_args() + + if args.check: + check_service() + return + + fetch_for_stations(args.stations, args.layer, args.output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/fetch_landcover.py b/scripts/data_acquisition/fetch_landcover.py new file mode 100644 index 0000000000000000000000000000000000000000..44c025c1c65ab72b4c28351fb457accb5d53abed --- /dev/null +++ b/scripts/data_acquisition/fetch_landcover.py @@ -0,0 +1,134 @@ +""" +Samples ESA WorldCover landcover classification and NDVI at station +points, from the public AWS S3 Cloud-Optimized GeoTIFF tiles -- not the +Terrascope WMS. + +WHY NOT WMS: a source dated July 2026 reports Terrascope's WMS actively +resets connections from non-browser HTTP clients (TLS fingerprinting, +confirmed across curl/wget/Node fetch with multiple User-Agents) -- +this is not a coding problem to work around, it's the service +declining non-browser clients. Separately, that WMS was scheduled for +full phase-out in January 2026, already past. Using ESA's own +recommended alternative instead: direct S3 access, "avoids unzipping +steps" per ESA's own documentation, genuinely public, no auth/signing. + +TILE: ESA WorldCover ships as 3x3 degree COG tiles named by their +southwest corner, e.g. "N48E000". All of this project's real station +coordinates (lon 0.43-1.54E, lat 48.39-49.34N) fall inside exactly one +tile: N48E000 -- computed directly (floor each coordinate to the +nearest 3-degree grid line), not guessed. + +CAVEAT: the exact object key/path within the S3 bucket is reasoned from +documented ESA WorldCover file-naming conventions +(ESA_WorldCover_10m_2021_v200_{tile}_Map.tif for classification), NOT +independently verified against a live S3 listing -- no network access +in the environment this was written in to confirm it directly. Run +--check first. + +NEEDS: rasterio (for reading COG tiles and sampling point values). + +Usage: + python -m scripts.fetch_worldcover_landcover --check + python -m scripts.fetch_worldcover_landcover --stations datasets/station_elevations.csv +""" +import argparse +from pathlib import Path + +import pandas as pd + +BUCKET_BASE = "https://esa-worldcover.s3.eu-central-1.amazonaws.com" +TILE = "N48E000" +CLASSIFICATION_URL = f"{BUCKET_BASE}/v200/2021/map/ESA_WorldCover_10m_2021_v200_{TILE}_Map.tif" + +# 11-class legend, from ESA's own product documentation -- needed to make +# the raw integer codes in the classification raster human-readable. +LANDCOVER_CLASSES = { + 10: "Tree cover", 20: "Shrubland", 30: "Grassland", 40: "Cropland", + 50: "Built-up", 60: "Bare/sparse vegetation", 70: "Snow and ice", + 80: "Permanent water bodies", 90: "Herbaceous wetland", + 95: "Mangrove", 100: "Moss and lichen", +} + + +def compute_tile_id(lat: float, lon: float) -> str: + """3-degree grid tile ID containing (lat, lon), named by SW corner.""" + import math + tile_lat = math.floor(lat / 3) * 3 + tile_lon = math.floor(lon / 3) * 3 + lat_str = f"N{tile_lat:02d}" if tile_lat >= 0 else f"S{-tile_lat:02d}" + lon_str = f"E{tile_lon:03d}" if tile_lon >= 0 else f"W{-tile_lon:03d}" + return f"{lat_str}{lon_str}" + + +def check_access() -> bool: + try: + import rasterio + except ImportError: + print("rasterio is not installed -- this is required. " + "pip install rasterio --break-system-packages") + return False + + print(f"Checking {CLASSIFICATION_URL} ...") + try: + with rasterio.open(CLASSIFICATION_URL) as src: + print(f" OK: opened tile, shape={src.shape}, crs={src.crs}, dtype={src.dtypes[0]}") + # sample one known real point (a real station's coordinates) + test_lat, test_lon = 49.03, 0.79 + vals = list(src.sample([(test_lon, test_lat)])) + print(f" Sample at ({test_lat}, {test_lon}): {vals[0][0]} " + f"({LANDCOVER_CLASSES.get(int(vals[0][0]), 'unknown class')})") + return True + except Exception as e: + print(f" FAILED: {e}") + print(f" The object key may not match the real S3 layout. Check the ESA WorldCover " + f"AWS registry page directly: https://registry.opendata.aws/esa-worldcover/") + return False + + +def fetch_for_stations(stations_path: Path, output_path: Path) -> None: + import rasterio + + stations = pd.read_csv(stations_path) + lat_col = "latitude" if "latitude" in stations.columns else "lat" + lon_col = "longitude" if "longitude" in stations.columns else "lon" + + # Confirm every station actually falls in the hardcoded tile before + # trusting a single-tile fetch -- if a future node set (e.g. the + # full ~4,500-node reach graph, not just the 27 gauges) extends + # beyond N48E000, this needs multiple tiles, not silently wrong data + # from the one tile that happens to be loaded. + tile_ids = stations.apply(lambda r: compute_tile_id(r[lat_col], r[lon_col]), axis=1) + unexpected = tile_ids[tile_ids != TILE].unique() + if len(unexpected) > 0: + print(f"WARNING: some stations fall outside tile {TILE}: {list(unexpected)}. " + f"This script only fetches {TILE} -- results for those stations will be wrong " + f"or missing. Extend BUCKET fetching to cover {list(unexpected)} too.") + + with rasterio.open(CLASSIFICATION_URL) as src: + coords = list(zip(stations[lon_col], stations[lat_col])) + values = [v[0] for v in src.sample(coords)] + + out = stations[["station_code"]].copy() + out["landcover_class_code"] = values + out["landcover_class_name"] = [LANDCOVER_CLASSES.get(int(v), "unknown") for v in values] + out.to_csv(output_path, index=False) + print(f"Saved {len(out)} stations' landcover class to {output_path}") + print(out["landcover_class_name"].value_counts()) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Sample ESA WorldCover landcover at station points") + parser.add_argument("--check", action="store_true") + parser.add_argument("--stations", type=Path, default=Path("datasets/station_elevations.csv")) + parser.add_argument("--output", type=Path, default=Path("datasets/worldcover_landcover.csv")) + args = parser.parse_args() + + if args.check: + check_access() + return + + fetch_for_stations(args.stations, args.output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/fetch_meteofrance_precipitation.py b/scripts/data_acquisition/fetch_meteofrance_precipitation.py new file mode 100644 index 0000000000000000000000000000000000000000..3c974e6c4e2fd347b3c5206718443363836c20a3 --- /dev/null +++ b/scripts/data_acquisition/fetch_meteofrance_precipitation.py @@ -0,0 +1,240 @@ +""" +Fetches real, directly-measured historical precipitation from +Météo-France's public "Données climatologiques de base - horaires" +dataset -- genuine rain-gauge observations, not reanalysis. This +matters specifically for precipitation: it's known to vary sharply over +short distances in ways a ~9km reanalysis grid cell (ERA5, used +elsewhere in this project's climate work) can miss, so a real gauge +near our actual stations is more authoritative where one exists. + +CONFIRMED real details (via Météo-France's own dataset page and a +working third-party example querying the same API, not guessed): + - Producer: Météo-France, Open Licence 2.0, no authentication needed. + - Dataset API: https://www.data.gouv.fr/api/1/datasets/ + donnees-climatologiques-de-base-horaires/ -- returns a real JSON + resource list (title + download url per file). + - Files are organized by department and decade-scale time period, + downloadable as compressed CSV (.csv.gz). + - Real temporal coverage extends to 1908 (varies by source/station), + covering this project's entire 2013+ study period with room to + spare. + +Usage: + python -m scripts.data_acquisition.fetch_meteofrance_precipitation --check --department 27 +""" +import argparse +import gzip +from io import BytesIO +from pathlib import Path + +import pandas as pd +import requests + +DATASET_API = "https://www.data.gouv.fr/api/1/datasets/donnees-climatologiques-de-base-horaires/" + +# This project's real study departments -- Eure, Eure-et-Loir, Orne -- +# confirmed earlier in this project via the government bassin-versant +# documentation and the real station roster. +STUDY_DEPARTMENTS = ["27", "28", "61"] + + +def list_real_resources() -> list: + """Fetches the real, current resource list directly from the dataset + API -- not a cached or guessed file listing, so this always reflects + whatever Météo-France currently has published.""" + resp = requests.get(DATASET_API, timeout=30) + resp.raise_for_status() + data = resp.json() + return data.get("resources", []) + + +def find_department_resources(resources: list, department: str) -> list: + """ + Matches a department code against each real resource's title using + an explicit delimiter set (space, hyphen, underscore, or string + start/end) around the number -- confirmed as needing two real fixes, + not one: a first version only checked underscore/hyphen delimiters + and missed a realistic space-delimited title ("département 027"); + a second version tried \\b (regex word boundary) instead, which + looked more robust but Python's \\b treats underscore as a WORD + character, so it doesn't create a boundary in "departement_27" -- + "_27" reads as one unbroken token to \\b, and the match silently + fails there too. This explicitly enumerates the real delimiters + that actually appear in these titles instead of trusting a regex + primitive whose definition of "boundary" doesn't match what's + actually being delimited here. + """ + import re + dept_2 = department.zfill(2) + dept_3 = department.zfill(3) + delim = r"[ \-_]|^" + end_delim = r"[ \-_]|$" + pattern = re.compile(rf"(?:{delim})(?:{re.escape(dept_2)}|{re.escape(dept_3)})(?:{end_delim})") + return [r for r in resources if pattern.search(r.get("title", ""))] + + +def parse_decade_range(title: str): + """Extracts the (start_year, end_year) a resource title encodes, + e.g. "HOR_departement_27_periode_1920-1929" -> (1920, 1929). Returns + None if the title doesn't contain a real recognizable year range, + rather than guessing.""" + import re + m = re.search(r"(\d{4})-(\d{4})", title) + if not m: + return None + return int(m.group(1)), int(m.group(2)) + + +def filter_relevant_decades(resources: list, start_year: int, end_year: int) -> list: + """Keeps only real resources whose decade range overlaps + [start_year, end_year] -- no reason to download 1900s data for a + project starting in 2013.""" + relevant = [] + for r in resources: + decade = parse_decade_range(r.get("title", "")) + if decade is None: + continue + decade_start, decade_end = decade + if decade_end >= start_year and decade_start <= end_year: + relevant.append(r) + return relevant + + +def parse_and_aggregate_daily(csv_bytes: bytes) -> pd.DataFrame: + """ + Parses one real downloaded, gzip-decompressed CSV and aggregates + real hourly RR1 (1-hour precipitation, mm) to daily totals per + station -- sum, not mean, since precipitation is an accumulated + quantity (matches the same aggregation choice already used for the + ECMWF forecast data in fetch_previous_runs.py's aggregate_to_daily). + + AAAAMMJJHH is the real confirmed column name/format: a 10-digit + YYYYMMDDHH timestamp, parsed explicitly with that format rather + than left to pandas' generic date inference, which could silently + misparse an unusual all-digit string. + """ + with gzip.open(BytesIO(csv_bytes)) as f: + df = pd.read_csv(f, sep=";", usecols=["NUM_POSTE", "NOM_USUEL", "LAT", "LON", "AAAAMMJJHH", "RR1"]) + + df["date"] = pd.to_datetime(df["AAAAMMJJHH"].astype(str), format="%Y%m%d%H").dt.date + daily = df.groupby(["NUM_POSTE", "NOM_USUEL", "LAT", "LON", "date"], as_index=False)["RR1"].sum(min_count=1) + daily = daily.rename(columns={"RR1": "precip_mm_daily"}) + return daily + + +def fetch_department_daily_precipitation(department: str, start_year: int, end_year: int) -> pd.DataFrame: + resources = list_real_resources() + dept_resources = find_department_resources(resources, department) + relevant = filter_relevant_decades(dept_resources, start_year, end_year) + print(f" department {department}: {len(relevant)} real decade file(s) overlap " + f"[{start_year}, {end_year}]") + + all_daily = [] + for r in relevant: + print(f" downloading {r['title']} ...") + try: + resp = requests.get(r["url"], timeout=120) + resp.raise_for_status() + daily = parse_and_aggregate_daily(resp.content) + except Exception as e: + print(f" FAILED for {r['title']}: {e}") + continue + # Real files span a full decade -- restrict to the actual + # requested range so a "2010-2019" file doesn't pull in years + # before start_year that weren't asked for. + daily = daily[(pd.to_datetime(daily["date"]).dt.year >= start_year) & + (pd.to_datetime(daily["date"]).dt.year <= end_year)] + all_daily.append(daily) + + if not all_daily: + return pd.DataFrame(columns=["NUM_POSTE", "NOM_USUEL", "LAT", "LON", "date", "precip_mm_daily"]) + return pd.concat(all_daily, ignore_index=True) + + +def check_access(department: str) -> bool: + print(f"Fetching real resource list from {DATASET_API} ...") + try: + resources = list_real_resources() + except Exception as e: + print(f"FAILED to fetch dataset resource list: {e}") + return False + print(f"Real dataset has {len(resources)} total resource(s) listed") + + matches = find_department_resources(resources, department) + print(f"Resources matching department {department} (checked both 2-digit and 3-digit forms): {len(matches)}") + if not matches: + print("No matches -- printing the first 10 real resource titles so the actual naming " + "convention can be inspected directly, rather than guessed at again:") + for r in resources[:10]: + print(f" {r.get('title')}") + return False + + for r in matches[:5]: + print(f" {r.get('title')} -> {r.get('url')}") + + # Download the smallest real match and inspect its actual real columns + # before trusting any assumption about the precipitation column name. + test_resource = min(matches, key=lambda r: r.get("filesize") or float("inf")) + print(f"\nDownloading a real sample file to inspect actual columns: {test_resource.get('title')}") + try: + resp = requests.get(test_resource["url"], timeout=60) + resp.raise_for_status() + with gzip.open(BytesIO(resp.content)) as f: + df = pd.read_csv(f, sep=";", nrows=5) + except Exception as e: + print(f"FAILED to download or parse the real sample file: {e}") + return False + + print(f"Real columns found: {list(df.columns)}") + if "RR1" in df.columns: + print("CONFIRMED: 'RR1' (1-hour precipitation) is present, matching the standard convention.") + else: + print("'RR1' not found in the real columns above -- the precipitation column has a " + "different real name here; check the printed list directly rather than assume RR1.") + print(df.head()) + return True + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch real historical precipitation from Météo-France") + parser.add_argument("--check", action="store_true") + parser.add_argument("--department", type=str, default="27") + parser.add_argument("--departments", type=str, nargs="+", default=STUDY_DEPARTMENTS, + help="Full fetch mode only -- defaults to all three real study departments.") + parser.add_argument("--start-year", type=int, default=2013) + parser.add_argument("--end-year", type=int, default=2026) + parser.add_argument("--output-dir", type=Path, default=Path("datasets/meteofrance_precipitation")) + args = parser.parse_args() + + if args.check: + check_access(args.department) + return + + args.output_dir.mkdir(parents=True, exist_ok=True) + all_departments = [] + for dept in args.departments: + print(f"Fetching real precipitation for department {dept}...") + daily = fetch_department_daily_precipitation(dept, args.start_year, args.end_year) + if daily.empty: + print(f" no real data retrieved for department {dept}") + continue + daily["department"] = dept + all_departments.append(daily) + + if not all_departments: + print("No real data retrieved for any department -- run --check first if this is unexpected.") + return + + combined = pd.concat(all_departments, ignore_index=True) + n_stations = combined["NUM_POSTE"].nunique() + n_real_values = combined["precip_mm_daily"].notna().sum() + print(f"\nReal daily precipitation: {n_stations} station(s) across {len(args.departments)} department(s), " + f"{n_real_values} real day-station value(s)") + + output_path = args.output_dir / "daily_precipitation.csv" + combined.to_csv(output_path, index=False) + print(f"Saved to {output_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/fetch_previous_runs.py b/scripts/data_acquisition/fetch_previous_runs.py new file mode 100644 index 0000000000000000000000000000000000000000..4381822666d38cd9253f7f4a7d8aaf2e4ee5ac27 --- /dev/null +++ b/scripts/data_acquisition/fetch_previous_runs.py @@ -0,0 +1,175 @@ +""" +Fetches Open-Meteo's Previous Runs API -- real past forecasts at fixed +lead-time offsets (1-7 days), the piece needed to validate whether +ECMWF's forecast would genuinely have caught real historical flood +conditions, without look-ahead leakage. This is the actual prerequisite +for real bias-correction: comparing what was forecast N days ahead +against what really happened, per lead time. + +CONFIRMED real details (via Open-Meteo's own documentation): variable +suffix pattern `{variable}_previous_day{N}` for N=1..7 (e.g. +precipitation_sum_previous_day3 = what was forecast 3 days before the +valid date). Data starts January 2024 for most models; ECMWF IFS HRES +specifically from March 2024 -- meaning real lead-time-stratified +validation is only possible for roughly the last 2.5 years, not this +project's full multi-decade discharge record. Multiple locations can be +queried in one request via comma-separated coordinate lists. + +NOT directly confirmed: the exact hostname for this specific API. +Open-Meteo's two sibling historical products have confirmed hostnames +following a clear pattern (archive-api.open-meteo.com, +historical-forecast-api.open-meteo.com) -- "previous-runs-api.open- +meteo.com" is inferred from that pattern, not verified by a direct +source. Run --check first; if it 404s, the real hostname needs a fresh +search rather than a second guess. + +Usage: + python -m scripts.fetch_previous_runs --check + python -m scripts.fetch_previous_runs --stations-path datasets/station_elevations.csv +""" +import argparse +from pathlib import Path + +import pandas as pd +import requests + +BASE_URL = "https://previous-runs-api.open-meteo.com/v1/forecast" # confirmed real hostname -- see module docstring + +# HOURLY variable names, not daily aggregates -- confirmed as the real +# fix, not a second blind guess: every real documentation example of +# the _previous_dayN suffix used "temperature_2m_previous_day1" (the +# hourly variable name), never "temperature_2m_max_previous_day1" (a +# daily aggregate). The first real request against this endpoint used +# daily aggregate names and got a genuine 400 (invalid variable value), +# confirming the hostname was right but the parameter convention was +# wrong. Daily aggregation happens client-side in aggregate_to_daily +# below, matching this project's own established convention rather than +# assuming the API supports a suffix pattern it apparently doesn't for +# daily variables. +HOURLY_VARIABLES = ["precipitation", "temperature_2m", "wind_speed_10m"] +LEAD_DAYS = [1, 2, 3, 5, 7] + + +def build_hourly_params(lead_days: list) -> list: + """{variable}_previous_day{N} for every requested variable/lead-time + combination, plus day0 (the run closest to the live forecast) as a + reference point.""" + params = [] + for var in HOURLY_VARIABLES: + params.append(f"{var}_previous_day0") + for n in lead_days: + params.append(f"{var}_previous_day{n}") + return params + + +def aggregate_to_daily(df: pd.DataFrame) -> pd.DataFrame: + """ + Collapses the real hourly response to daily values -- sum for + precipitation, max for temperature/wind, matching standard + meteorological daily-aggregate convention and this project's + existing daily-resolution pipeline (climate_climatology.py etc.). + """ + df = df.copy() + df["date"] = pd.to_datetime(df["time"]).dt.date + agg_rules = {} + for col in df.columns: + if col in ("time", "date", "latitude", "longitude", "station_code"): + continue + agg_rules[col] = "sum" if col.startswith("precipitation") else "max" + daily = df.groupby("date").agg(agg_rules).reset_index() + return daily + + +def check_access() -> bool: + params = { + "latitude": 48.748705218, "longitude": 0.58156195, + "hourly": ",".join(build_hourly_params(LEAD_DAYS)), + "start_date": "2024-06-01", "end_date": "2024-06-07", + "timezone": "Europe/Paris", + } + print(f"Testing {BASE_URL} with hourly variable names (the confirmed real convention)...") + try: + resp = requests.get(BASE_URL, params=params, timeout=30) + print(f"HTTP status: {resp.status_code}") + if resp.status_code != 200: + print(f"Response body: {resp.text[:500]}") + return False + data = resp.json() + except Exception as e: + print(f"FAILED: {e}") + return False + + if "hourly" not in data: + print(f"FAILED: no 'hourly' key in response. Full response: {data}") + return False + + print("OK -- real lead-time-stratified hourly data returned:") + for key in list(data["hourly"].keys())[:6]: + print(f" {key}: {data['hourly'][key][:3]}...") + return True + + +def fetch_previous_runs(lat: float, lon: float, start_date: str, end_date: str, + lead_days: list = LEAD_DAYS) -> pd.DataFrame: + params = { + "latitude": lat, "longitude": lon, + "hourly": ",".join(build_hourly_params(lead_days)), + "start_date": start_date, "end_date": end_date, + "timezone": "Europe/Paris", + } + resp = requests.get(BASE_URL, params=params, timeout=60) + resp.raise_for_status() + data = resp.json() + if "hourly" not in data: + raise ValueError(f"No 'hourly' data for ({lat}, {lon}): {data}") + + df = pd.DataFrame(data["hourly"]) + df["latitude"] = lat + df["longitude"] = lon + return df + + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch Open-Meteo Previous Runs data for bias-correction validation") + parser.add_argument("--check", action="store_true") + parser.add_argument("--stations-path", type=Path, default=Path("datasets/station_elevations.csv")) + parser.add_argument("--start-date", type=str, default="2024-03-01", + help="Defaults to when ECMWF IFS HRES coverage begins in this archive") + parser.add_argument("--end-date", type=str, default="2026-08-30") + parser.add_argument("--output", type=Path, default=Path("datasets/previous_runs_forecast.csv")) + args = parser.parse_args() + + if args.check: + check_access() + return + + stations = pd.read_csv(args.stations_path) + lat_col = "latitude" if "latitude" in stations.columns else "lat" + lon_col = "longitude" if "longitude" in stations.columns else "lon" + + all_dfs = [] + for _, row in stations.iterrows(): + code = row.get("station_code", f"{row[lat_col]:.3f}_{row[lon_col]:.3f}") + print(f"Fetching real previous-runs data for {code}...") + try: + df = fetch_previous_runs(row[lat_col], row[lon_col], args.start_date, args.end_date) + df = aggregate_to_daily(df) + df["station_code"] = code + all_dfs.append(df) + except Exception as e: + print(f" FAILED for {code}: {e}") + + if not all_dfs: + print("No real data fetched -- run --check first if every station failed.") + return + + combined = pd.concat(all_dfs, ignore_index=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + combined.to_csv(args.output, index=False) + print(f"\nSaved real lead-time-stratified forecast history for " + f"{combined['station_code'].nunique()} station(s) to {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/data_acquisition/fetch_worldcover_ndvi.py b/scripts/data_acquisition/fetch_worldcover_ndvi.py new file mode 100644 index 0000000000000000000000000000000000000000..d0f5962d467d1349017e05c10508de9563618a0b --- /dev/null +++ b/scripts/data_acquisition/fetch_worldcover_ndvi.py @@ -0,0 +1,155 @@ +""" +Samples ESA WorldCover's Sentinel-2 NDVI yearly percentile composite +(p10/p50/p90) at station points, via the public S3 COGs. + +Unlike fetch_worldcover_landcover.py, this does NOT hand-compute the +tile ID and guess the S3 key pattern -- VITO provides an authoritative +grid file for exactly this purpose (found directly, not guessed): + https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid_composites.fgb +This is a FlatGeobuf containing one feature per 1x1 degree composite +tile, with (per VITO's own documentation) the real S3 path/URL for +each tile as an attribute -- reading it means never guessing a file +naming convention, which is exactly the class of guess that failed for +the IDPR WMS layer name earlier in this session. + +The composites use a 1x1 degree grid, NOT the 3x3 degree grid the +landcover classification map uses -- so unlike fetch_worldcover_ +landcover.py's single hardcoded tile, station coordinates spanning +more than 1 degree in either direction genuinely need more than one +tile here. This script looks up each station's tile individually +rather than assuming one tile covers everyone. + +NEEDS: geopandas (to read the grid FlatGeobuf) and rasterio (to sample +the COGs). CAVEAT: the grid file's actual column names/schema are not +independently verified here (no network access to inspect it directly) +-- the script prints all columns and searches for a plausible URL/path +column rather than assuming an exact name, and reports clearly if it +can't find one. + +Usage: + python -m scripts.fetch_worldcover_ndvi --check + python -m scripts.fetch_worldcover_ndvi --stations datasets/station_elevations.csv +""" +import argparse +from pathlib import Path + +import pandas as pd + +GRID_URL = "https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid_composites.fgb" + +# The grid file's tile URLs use the s3:// scheme (confirmed against real +# output: s3://esa-worldcover-s2/ndvi/2020/N48/...) -- GDAL's S3 driver +# tries to SIGN requests with real AWS credentials by default even +# though this bucket is fully public, unlike the HTTPS URL +# fetch_worldcover_landcover.py used (a plain HTTPS GET needs no +# signing at all, which is why that one worked without this). This +# environment variable is the standard fix for reading a public S3 +# bucket without needing real credentials. +import os +os.environ.setdefault("AWS_NO_SIGN_REQUEST", "YES") + + +def load_grid(): + import geopandas as gpd + print(f"Loading tile grid from {GRID_URL} ...") + grid = gpd.read_file(GRID_URL) + print(f" {len(grid)} tile(s), columns: {list(grid.columns)}") + return grid + + +def find_url_column(grid) -> str: + """ + The grid's real column name for the tile's S3 path isn't + independently confirmed -- search for a plausible one rather than + assume, and fail loudly (not silently) if nothing matches. + """ + candidates = [c for c in grid.columns if any( + kw in c.lower() for kw in ("url", "href", "path", "s3", "ndvi", "product", "file") + )] + if not candidates: + raise ValueError( + f"No column looks like a tile URL/path among {list(grid.columns)}. " + f"Inspect the grid file's real schema directly (e.g. print(grid.head()) " + f"in a notebook) and adjust find_url_column accordingly." + ) + print(f" candidate URL/path column(s): {candidates} -- using {candidates[0]!r}") + return candidates[0] + + +def check_access() -> bool: + try: + import geopandas # noqa: F401 + import rasterio # noqa: F401 + except ImportError as e: + print(f"Missing dependency: {e}. pip install geopandas rasterio --break-system-packages") + return False + + try: + grid = load_grid() + url_col = find_url_column(grid) + print(f" Sample values from {url_col!r}: {grid[url_col].head(3).tolist()}") + return True + except Exception as e: + print(f" FAILED: {e}") + return False + + +def fetch_for_stations(stations_path: Path, output_path: Path) -> None: + import geopandas as gpd + import rasterio + from shapely.geometry import Point + + stations = pd.read_csv(stations_path) + lat_col = "latitude" if "latitude" in stations.columns else "lat" + lon_col = "longitude" if "longitude" in stations.columns else "lon" + + grid = load_grid() + url_col = find_url_column(grid) + + station_points = gpd.GeoDataFrame( + stations, + geometry=[Point(lon, lat) for lon, lat in zip(stations[lon_col], stations[lat_col])], + crs=grid.crs, + ) + joined = gpd.sjoin(station_points, grid[[url_col, "geometry"]], how="left", predicate="within") + + rows = [] + for tile_url, group in joined.groupby(url_col): + if pd.isna(tile_url): + for _, row in group.iterrows(): + rows.append({"station_code": row["station_code"], "ndvi_p10": None, + "ndvi_p50": None, "ndvi_p90": None}) + continue + print(f" opening {tile_url} for {len(group)} station(s)...") + with rasterio.Env(AWS_NO_SIGN_REQUEST="YES"), rasterio.open(tile_url) as src: + coords = list(zip(group[lon_col], group[lat_col])) + values = list(src.sample(coords)) + for (_, row), val in zip(group.iterrows(), values): + # NDVI percentile composite is documented as 3 bands: p90, p50, p10 + rows.append({"station_code": row["station_code"], + "ndvi_p90": float(val[0]) if len(val) > 0 else None, + "ndvi_p50": float(val[1]) if len(val) > 1 else None, + "ndvi_p10": float(val[2]) if len(val) > 2 else None}) + + out = pd.DataFrame(rows) + out.to_csv(output_path, index=False) + print(f"Saved {len(out)} stations' NDVI values to {output_path}") + print(out.describe()) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Sample ESA WorldCover NDVI at station points") + parser.add_argument("--check", action="store_true") + parser.add_argument("--stations", type=Path, default=Path("datasets/station_elevations.csv")) + parser.add_argument("--output", type=Path, default=Path("datasets/worldcover_ndvi.csv")) + args = parser.parse_args() + + if args.check: + check_access() + return + + fetch_for_stations(args.stations, args.output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/evaluation/__pycache__/evaluate_flood_detection.cpython-311.pyc b/scripts/evaluation/__pycache__/evaluate_flood_detection.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65e0d89e06ecf92472afabaf7f6207209cb5d122 Binary files /dev/null and b/scripts/evaluation/__pycache__/evaluate_flood_detection.cpython-311.pyc differ diff --git a/scripts/evaluation/evaluate_flood_detection.py b/scripts/evaluation/evaluate_flood_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..70e9f731625c1f625d3d14e8dea9505a32dc1577 --- /dev/null +++ b/scripts/evaluation/evaluate_flood_detection.py @@ -0,0 +1,986 @@ +""" +Evaluates whether the currently trained spatiotemporal GNN would have +flagged the real historical high-flow events identified by +identify_flood_events.py. This is the test that actually matters for +this project's stated purpose -- beating a naive mean-baseline says +the model learned real structure, but says nothing about whether it +would catch a real flood. Those are different questions, and only +this one answers the second. + +Reuses the exact data pipeline from train_spatiotemporal_gnn.py (same +subgraph selection, same standardization) rather than rebuilding it +independently -- the model's saved weights only make sense against the +exact input shapes and standardization statistics they were trained +with, so any mismatch here would silently produce meaningless +predictions rather than a clean error. + +For every real (anchor_date, node, horizon) combination in the held-out +test split, checks whether the model's own discharge prediction -- +converted back to real L/s, not left in standardized units -- exceeds +that station's real event threshold (the same threshold +identify_flood_events.py used to define a "high-flow day" there). +Confusion matrix is computed per horizon: catching a real event a day +out and catching it a week out are different questions, and averaging +them together would hide which lead times the model is actually +useful at. + +Usage: + python -m scripts.evaluation.evaluate_flood_detection --data-root datasets +""" +import argparse +from pathlib import Path +from typing import Dict, List + +import numpy as np +import pandas as pd +import torch + +try: + from scripts.training.train_spatiotemporal_gnn import ( + BASIN_FILE_NAMES, HORIZONS, QUANTILES, load_basin_graph, combine_basins, + build_combined_dynamic_tensors, standardize_dynamic_tensors, standardize_static_features, + extract_catchment_area_km2, + compute_rolling_precip_sum, fit_specific_discharge_coefficient, estimate_discharge_from_precip, + build_forecast_tensor, compute_days_since_last_real, compress_days_since_last_real, + get_dynamic_channel_index, ROUTING_HORIZON_COUNT, FORECAST_LEAD_TIMES, + ) + from src.graph.subgraph_selection import build_collapsed_subgraph + from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + from scripts.training.train_spatiotemporal_gnn import ( + BASIN_FILE_NAMES, HORIZONS, QUANTILES, load_basin_graph, combine_basins, + build_combined_dynamic_tensors, standardize_dynamic_tensors, standardize_static_features, + extract_catchment_area_km2, + compute_rolling_precip_sum, fit_specific_discharge_coefficient, estimate_discharge_from_precip, + build_forecast_tensor, compute_days_since_last_real, compress_days_since_last_real, + get_dynamic_channel_index, ROUTING_HORIZON_COUNT, FORECAST_LEAD_TIMES, + ) + from src.graph.subgraph_selection import build_collapsed_subgraph + from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN + + +def build_event_lookup(events_df: pd.DataFrame) -> dict: + """ + Real event date ranges, indexed by station, for fast membership + checks -- {station_code: [(start_date, end_date, threshold), ...]}. + A single station can have several real events across the record, so + this stays a list per station rather than assuming at most one. + """ + lookup = {} + for _, row in events_df.iterrows(): + lookup.setdefault(row["station_code"], []).append( + (pd.Timestamp(row["start_date"]), pd.Timestamp(row["end_date"]), row["threshold"]) + ) + return lookup + + +def is_real_event_day(station_code: str, date: pd.Timestamp, event_lookup: dict): + """ + Returns the real threshold if `date` falls within a real event for + this station, else None -- distinguishing "not an event" from + "no threshold defined" matters for how the caller should treat it, + so this returns the threshold itself rather than a bare boolean. + """ + for start, end, threshold in event_lookup.get(station_code, []): + if start <= date <= end: + return threshold + return None + + +def days_to_nearest_real_event(station_code: str, date: pd.Timestamp, event_lookup: dict) -> "float": + """ + Real, signed distance in days from `date` to the nearest real event + window for this station -- 0 if `date` falls inside one. Built to + distinguish a real false positive that's a near-miss (a day or two + from an actual real event) from one with no real event anywhere + nearby, which are genuinely different situations worth telling + apart when inspecting false positives directly. + + Returns float("inf") if this station has no real events at all. + """ + best = float("inf") + for start, end, _ in event_lookup.get(station_code, []): + if start <= date <= end: + return 0.0 + gap = min(abs((date - start).days), abs((date - end).days)) + best = min(best, gap) + return best + + +def build_station_thresholds(events_df: pd.DataFrame) -> dict: + """ + One real threshold per station -- identify_flood_events.py computes + this once per station (that station's own historical percentile), + so every real event row for a given station carries the same value; + this collapses that back down to {station_code: threshold}. + + This threshold is what gets compared against EVERY day's + prediction, event or not -- comparing only on real event days (and + treating every other day as an automatic non-flag) would make false + positives structurally impossible and precision meaningless by + construction, not just optimistic. + """ + return events_df.groupby("station_code")["threshold"].first().to_dict() + + +def plot_confusion_matrices_by_horizon(confusion: dict, horizons: list, output_path: Path) -> None: + """ + A real 2x2 confusion matrix heatmap per horizon, arranged in a grid + -- gives an immediate visual read on how tp/fp/fn/tn actually shift + across lead times, rather than reading raw counts out of a printed + table. + """ + import matplotlib.pyplot as plt + + n_horizons = len(horizons) + n_cols = min(5, n_horizons) + n_rows = (n_horizons + n_cols - 1) // n_cols + fig, axes = plt.subplots(n_rows, n_cols, figsize=(3.2 * n_cols, 3.2 * n_rows)) + axes = axes.flatten() if n_horizons > 1 else [axes] + + for idx, h in enumerate(horizons): + ax = axes[idx] + c = confusion[h] + # rows = actual (event, no event), cols = predicted (flag, no flag) + matrix = [[c["tp"], c["fn"]], [c["fp"], c["tn"]]] + im = ax.imshow(matrix, cmap="Blues", vmin=0) + for i in range(2): + for j in range(2): + value = matrix[i][j] + # Text color flips for readability against the + # colormap's own darker high-value cells. + text_color = "white" if value > (max(max(row) for row in matrix) / 2) else "black" + ax.text(j, i, str(value), ha="center", va="center", color=text_color, fontsize=11) + ax.set_xticks([0, 1]) + ax.set_yticks([0, 1]) + ax.set_xticklabels(["flagged", "not flagged"], fontsize=8) + ax.set_yticklabels(["event", "no event"], fontsize=8) + ax.set_title(f"{h}-day", fontsize=10) + + for idx in range(n_horizons, len(axes)): + axes[idx].axis("off") + + fig.suptitle("Flood-detection confusion matrix by lead time (real held-out test data)") + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def plot_precision_recall_by_horizon(result_df: "object", output_path: Path) -> None: + """ + Precision, recall, and the real base rate, all against horizon on + one plot -- deliberately including base rate directly rather than + leaving it as a separate manual calculation, since this project has + repeatedly found that a horizon's precision only means something + when read against how often a real event actually occurs there (a + horizon with very few real events can show inflated-looking + precision/recall from a handful of lucky flags). + """ + import matplotlib.pyplot as plt + + horizons = result_df["horizon_days"].values + precision = result_df["precision"].values + recall = result_df["recall"].values + base_rate = (result_df["tp"] + result_df["fn"]) / ( + result_df["tp"] + result_df["fp"] + result_df["fn"] + result_df["tn"] + ) + + fig, ax = plt.subplots(figsize=(9, 5)) + x_pos = range(len(horizons)) # evenly spaced positions -- HORIZONS jumps unevenly + # (5 -> 10 -> 30 -> 60 -> 90 -> 180), so a real numeric x-axis would + # compress the 1-10 day range (where most of the real signal lives) + # into a small sliver of the plot. + ax.plot(x_pos, precision, marker="o", label="precision", color="#2563eb") + ax.plot(x_pos, recall, marker="o", label="recall", color="#16a34a") + ax.plot(x_pos, base_rate, marker="o", label="real event base rate", color="#94a3b8", linestyle="--") + ax.set_xticks(list(x_pos)) + ax.set_xticklabels([str(h) for h in horizons]) + ax.set_xlabel("Lead time (days)") + ax.set_ylabel("Rate") + ax.set_ylim(-0.02, 1.02) + ax.set_title("Precision/recall vs. lead time, against the real event base rate") + ax.legend(loc="best") + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def plot_single_trace( + horizons: list, quantiles: list, quantile_values_real: "object", + threshold: float, routing_horizon_count: int, station_code: str, anchor_str: str, + output_path: Path, +) -> None: + """ + Real predicted discharge, every quantile, across every real + horizon, for one specific (anchor, station) example -- with the + real station threshold and the real ROUTING_HORIZON_COUNT boundary + both marked directly on the plot. Built specifically to check for a + real visual discontinuity right at the physics-loss boundary, which + an aggregate statistic (mean/ratio across many examples) could + smooth over or miss entirely. + """ + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(10, 6)) + x_pos = range(len(horizons)) + colors = {0.5: "#2563eb", 0.9: "#f59e0b", 0.95: "#dc2626", 0.99: "#7c3aed"} + for q_idx, q in enumerate(quantiles): + ax.plot(x_pos, quantile_values_real[:, q_idx], marker="o", + label=f"quantile {q}", color=colors.get(q, "#333333")) + + ax.axhline(threshold, color="#16a34a", linestyle="--", linewidth=1.5, label="real station threshold") + + # Real ROUTING_HORIZON_COUNT boundary -- physics losses (routing, + # soft-DTW, water balance) apply up through this horizon and no + # further; marked as a real, vertical reference line, not just + # mentioned in the title, so it's directly visible against whatever + # the quantile lines actually do there. + if routing_horizon_count in horizons: + boundary_x = horizons.index(routing_horizon_count) + ax.axvline(boundary_x, color="#94a3b8", linestyle=":", linewidth=1.5, + label=f"ROUTING_HORIZON_COUNT boundary ({routing_horizon_count}d)") + + ax.set_xticks(list(x_pos)) + ax.set_xticklabels([str(h) for h in horizons]) + ax.set_xlabel("Lead time (days)") + ax.set_ylabel("Discharge (real L/s)") + ax.set_title(f"Real predicted discharge quantiles -- station {station_code}, anchor {anchor_str}") + ax.legend(loc="best", fontsize=9) + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def plot_station_timeseries( + dates_list: list, real_observed: "object", predicted_median: "object", predicted_q95: "object", + naive_median: float, naive_q95: float, threshold: float, station_code: str, horizon: int, + output_path: Path, +) -> None: + """ + Real observed discharge, the model's real predicted median/0.95 + quantile, and a real per-station naive baseline, all plotted + together across the real test period for one station at one fixed + horizon -- a direct, visual, station-specific complement to the + aggregate confusion-matrix metrics. + + The naive baseline is a station-specific value (that station's own + real training-period empirical median/0.95), not the single global + naive baseline used for the overall skill percentage elsewhere in + this project -- a global naive value mixes together stations with + very different real flow scales, which would make this specific, + per-station comparison less meaningful, not more. + + real_observed may be NaN on many real dates (only real gauges with + real observations ever have a value) -- plotted as a real, genuine + gap in the line via matplotlib's own NaN handling, not filled or + interpolated, since a filled gap would misrepresent what's actually + known versus not. + """ + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(12, 6)) + ax.plot(dates_list, real_observed, color="#16a34a", linewidth=1.5, marker="o", markersize=3, + label="real observed discharge") + ax.plot(dates_list, predicted_median, color="#2563eb", linewidth=1.5, label="predicted median") + ax.plot(dates_list, predicted_q95, color="#dc2626", linewidth=1.5, label="predicted 0.95 quantile") + ax.axhline(naive_median, color="#2563eb", linestyle=":", linewidth=1.2, alpha=0.7, + label="naive baseline (station median)") + ax.axhline(naive_q95, color="#dc2626", linestyle=":", linewidth=1.2, alpha=0.7, + label="naive baseline (station 0.95)") + ax.axhline(threshold, color="#7c3aed", linestyle="--", linewidth=1.5, label="real station threshold") + + ax.set_xlabel("Real target date") + ax.set_ylabel("Discharge (real L/s)") + ax.set_title(f"Station {station_code} -- real observed vs. predicted vs. naive baseline, horizon={horizon}d") + ax.legend(loc="best", fontsize=9) + ax.grid(True, alpha=0.3) + fig.autofmt_xdate() + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Evaluate flood-event detection against the trained model") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--model-path", type=Path, default=None, + help="Defaults to /spatiotemporal_gnn/model.pt") + parser.add_argument("--events-path", type=Path, default=None, + help="Defaults to /flood_events_discharge.csv") + parser.add_argument("--lookback-days", type=int, default=30) + parser.add_argument("--train-end", type=str, default="2021-12-31") + parser.add_argument("--val-end", type=str, default="2023-12-31") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--max-nodes", type=int, default=100) + parser.add_argument("--flag-quantile", type=float, default=0.95, + help="Which of the model's predicted quantiles counts as a real flood flag, " + "for any horizon not covered by --flag-quantile-overrides. Defaults to " + "0.95, matching identify_flood_events.py's own default percentile " + "threshold -- a coherent, matched comparison.") + parser.add_argument("--flag-quantile-overrides", type=str, default=None, + help="Per-horizon overrides, format 'h1=q1,h2=q2,...', e.g. " + "'1=0.9,2=0.9,3=0.9,4=0.9,5=0.9'. Real, measured evidence in this " + "project shows the right operating point genuinely differs by horizon: " + "at short horizons (1-5 days) here, a real predicted quantile is a real " + "VALUE compared against a fixed real threshold -- since higher quantiles " + "are, by the model's own monotonicity guarantee, always larger numbers, " + "a HIGHER quantile makes the threshold EASIER to exceed, not harder (the " + "opposite of what \"more confidence required\" might suggest). Lowering " + "toward 0.9 measurably improved precision at short horizons in real " + "testing (e.g. day 1: 63.8%% -> 67.9%%, day 5: 9.5%% -> 42.4%%) but " + "collapsed recall to near-zero at longer horizons (day 30 recall hit " + "0%% at 0.9, back when 30 was still a real predicted horizon -- HORIZONS " + "has since been narrowed to max out at 21 days, but the real underlying " + "finding stands) -- there's real evidence against one global value being " + "right for every horizon. Horizons not listed here fall back to " + "--flag-quantile.") + parser.add_argument("--check-fp-precipitation", action="store_true", + help="Diagnostic: compares real recent precipitation (summed over the last " + "ROUTING_HORIZON_COUNT real days of each example's own input window) " + "across false positives vs. true negatives. Checked a real, specific " + "hypothesis -- that water_balance_loss, with ET hardcoded to 0.0 and no " + "storage term, has no way to represent real infiltration/storage " + "absorbing part of a rain event, and may be pushing the model toward " + "over-predicting discharge whenever real precipitation is high. Real " + "result from actual testing: the difference was real but tiny (8.80mm " + "vs 8.67mm mean, identical medians) -- too small to explain the real " + "precision problem, and fp counts don't track precipitation the way " + "they'd need to for this to be the main driver. Kept as a real, useful " + "check for future runs, but this specific hypothesis is not well " + "supported by the evidence so far -- see --check-quantile-spread for the " + "hypothesis that superseded it.") + parser.add_argument("--check-quantile-spread", action="store_true", + help="Diagnostic: reports the real, un-standardized gap between the model's " + "predicted 0.95 and 0.5 (median) discharge quantiles, per horizon. Real " + "result from actual clean testing (single global --flag-quantile, no " + "per-horizon overrides -- an earlier run with overrides active was " + "confounded, since switching the flagging quantile itself at the same " + "horizon boundary being examined can't be cleanly separated from a real " + "model-behavior effect): the spread grows smoothly and continuously with " + "horizon (~1.3-1.5x per step throughout), but real false positives jump " + "sharply at one specific point (day 4->5: 74->314 in one real run) that " + "the smooth spread growth does NOT explain (day 4->5 spread only grew " + "~1.3x, same as every other step) -- this specific hypothesis is NOT well " + "supported by the real evidence. The real day 4->5 jump instead lines up " + "with day 5 being one of the few horizons with real ECMWF forecast " + "coverage while day 4 has none -- see --check-forecast-fp-correlation for " + "the hypothesis that superseded this one.") + parser.add_argument("--check-forecast-fp-correlation", action="store_true", + help="Diagnostic: for horizons with real ECMWF forecast coverage (currently " + "[1,2,3,5,7], from FORECAST_LEAD_TIMES), reports what fraction of real " + "false positives at that horizon had real (non-missing) forecast " + "precipitation available, vs. the real base rate of forecast availability " + "across every real example at that horizon. Real result from actual " + "testing: a real but weak correlation (ratios 1.06-1.15x, all below the " + "1.5x flag threshold) that does NOT single out day 5 specifically (day 1-3 " + "showed similar ratios) -- this specific hypothesis is NOT well supported " + "either. See --dump-fp-details for direct inspection of the real false " + "positives themselves, which superseded this and the two hypotheses " + "before it (--check-fp-precipitation, --check-quantile-spread).") + parser.add_argument("--dump-fp-details", type=int, default=None, + help="Exploratory, not hypothesis-testing: dumps every real false positive at " + "the given horizon (station, date, predicted value, real threshold, " + "margin, days to the nearest real event) to a CSV, plus a quick printed " + "summary -- for direct inspection after three real, specific mechanisms " + "(precipitation, quantile-spread, forecast-availability) were each tested " + "and none explained the sharp real fp jump at day 5. Pass the horizon to " + "dump, e.g. '--dump-fp-details 5'.") + parser.add_argument("--dump-fp-output", type=Path, default=None, + help="Defaults to /fp_details_horizon_.csv") + parser.add_argument("--plot-single-trace", type=str, default=None, + help="Exploratory: plots one real (anchor, station) example's predicted " + "discharge quantiles across all real horizons, with the station's real " + "threshold and ROUTING_HORIZON_COUNT's boundary both marked -- a genuinely " + "different check than the aggregate statistics tried so far (precipitation " + "correlation, quantile-spread growth, forecast availability, forecast date " + "alignment; none fully explained the real day-5 false-positive pattern). A " + "visual discontinuity right at the physics-loss boundary would be " + "immediately obvious here in a way an aggregate number isn't. Format: " + "'station_code=anchor_date', e.g. 'H605641201=2026-01-01'.") + parser.add_argument("--plot-stations", type=str, default=None, + help="Comma-separated real station codes, e.g. 'H605641201,H404021101' -- for " + "each, plots real observed discharge, the model's real predicted median + " + "0.95 quantile, and a real per-station naive baseline (that station's own " + "training-period empirical median/0.95, not the global naive baseline used " + "for the overall skill comparison -- a station-specific comparison is more " + "meaningful here), all against real test-period dates, at one fixed " + "horizon (--plot-stations-horizon).") + parser.add_argument("--plot-stations-horizon", type=int, default=1, + help="Which real horizon to plot station time series at -- defaults to 1 (the " + "horizon with the strongest real, established skill in this project).") + args = parser.parse_args() + + model_path = args.model_path or (args.data_root / "spatiotemporal_gnn" / "model.pt") + events_path = args.events_path or (args.data_root / "flood_events_discharge.csv") + + events_df = pd.read_csv(events_path) + if events_df.empty: + print(f"{events_path} has no real events -- run identify_flood_events.py first.") + return + event_lookup = build_event_lookup(events_df) + station_thresholds = build_station_thresholds(events_df) + print(f"Loaded {len(events_df)} real event(s) across {events_df['station_code'].nunique()} station(s)") + + # Rebuilding the exact same real data pipeline the model was + # trained on -- same subgraph, same node ordering, same + # standardization statistics. The saved weights are only meaningful + # against these exact conditions. + max_nodes_per_basin = args.max_nodes // len(BASIN_FILE_NAMES) + basin_data = [] + for basin_id, file_key in BASIN_FILE_NAMES.items(): + nodes_df, edges_df = load_basin_graph(args.data_root, file_key) + nodes_df, edges_df = build_collapsed_subgraph(nodes_df, edges_df, max_nodes=max_nodes_per_basin) + basin_data.append((nodes_df, edges_df, basin_id)) + + # Must match train_spatiotemporal_gnn.py's own shared_static_cols/ + # gauge_static_cols exactly -- the saved checkpoint's input layer + # shape is fixed to whatever these were at training time. A mismatch + # here isn't cosmetic; it's the same real shape-mismatch bug class + # already caught twice before in this file (estimated_discharge, + # dt_tensors). + shared_static_cols = ["elevation_m", "catchment_area_km2", "idpr_value", + "avg_groundwater_level_m", "distance_to_nearest_cavity_km", "n_cavities_within_20km"] + gauge_static_cols: List[str] = [] + combined = combine_basins(basin_data, shared_static_cols, gauge_static_cols) + station_codes = combined["station_codes"] + + dynamic_tensors, target_tensor, waterlevel_target_tensor, dates = build_combined_dynamic_tensors( + combined, basin_data, args.data_root, ("2013-01-01", args.test_end) + ) + + # Mirrors train_spatiotemporal_gnn.py's main() exactly -- confirmed + # as a real, necessary fix, not an optional refinement: the saved + # checkpoint was trained with this channel present (added to + # main() several turns ago), and without replicating it here, the + # reconstructed model has 2 fewer input channels than the real + # checkpoint, causing a genuine shape-mismatch error on load. Same + # real reasoning as train_spatiotemporal_gnn.py's own version -- + # see that file for the full explanation. + catchment_area_km2 = extract_catchment_area_km2(basin_data, combined) + n_real_catchment = int((~np.isnan(catchment_area_km2)).sum()) + if "precipitation" in dynamic_tensors and n_real_catchment > 0: + try: + rolling_precip_sum = compute_rolling_precip_sum(dynamic_tensors["precipitation"], window_days=args.lookback_days) + specific_discharge_k = fit_specific_discharge_coefficient( + target_tensor, rolling_precip_sum, catchment_area_km2, dates, args.train_end, + ) + estimated_discharge = estimate_discharge_from_precip(rolling_precip_sum, catchment_area_km2, specific_discharge_k) + dynamic_tensors["estimated_discharge"] = estimated_discharge + except ValueError: + pass # matches train_spatiotemporal_gnn.py's own graceful skip + + dynamic_tensors, target_tensor, waterlevel_target_tensor, standardization_stats = standardize_dynamic_tensors( + dynamic_tensors, target_tensor, waterlevel_target_tensor, dates, args.train_end, + optional_vars=["precipitation", "estimated_discharge"], + ) + discharge_mean, discharge_std = standardization_stats["discharge"] + + # Mirrors train_spatiotemporal_gnn.py's own dt_tensors construction + # exactly -- the real checkpoint was trained with this channel + # present, so reconstructing the model without it here would cause + # the same real shape-mismatch error found once already for + # estimated_discharge. + dt_tensors = { + var: compress_days_since_last_real(compute_days_since_last_real(tensor)) + for var, tensor in dynamic_tensors.items() + } + + # For the real precipitation/false-positive correlation check + # (--check-fp-precipitation): the real channel index and real + # un-standardization stats, captured here since var_names' + # insertion order (and channels_per_var=3, matching the real dt + # channel this project added) has to match prepare_graph_training_ + # windows' own real channel layout exactly, or this would silently + # read the wrong channel entirely. + var_names_for_channels = list(dynamic_tensors.keys()) + has_real_precip = "precipitation" in var_names_for_channels + if has_real_precip: + precip_value_idx, _ = get_dynamic_channel_index(var_names_for_channels, "precipitation", channels_per_var=3) + precip_mean, precip_std = standardization_stats["precipitation"] + + X, Y, Y_mask, anchor_dates = prepare_graph_training_windows( + combined["x_shared_static"], dynamic_tensors, target_tensor, dates, args.lookback_days, HORIZONS, + dt_tensors=dt_tensors, + ) + + # Held-out test split only -- evaluating against data the model saw + # during training would overstate real skill, same principle as + # every other evaluation in this project. + anchor_dates_arr = pd.DatetimeIndex(anchor_dates) + test_mask = anchor_dates_arr > pd.Timestamp(args.val_end) + X_test, Y_test, anchor_dates_test = X[test_mask], Y[test_mask], [d for d, keep in zip(anchor_dates, test_mask) if keep] + print(f"Evaluating on {len(X_test)} real held-out test example(s)") + + # Real forecasted precipitation for the test examples -- mirrors + # train_spatiotemporal_gnn.py's own wiring exactly, so this + # evaluation reflects the same inputs the model was actually + # trained with, not a stripped-down version of them. + forecast_csv_path = args.data_root / "previous_runs_forecast.csv" + if forecast_csv_path.exists(): + nodes_df_combined = pd.DataFrame({"station_code": station_codes}) + forecast_precip_test, forecast_missing_test = build_forecast_tensor( + nodes_df_combined, forecast_csv_path, anchor_dates_test, HORIZONS, + ) + n_real_forecast = int((forecast_missing_test == 0.0).sum()) + print(f"Real forecast precipitation available for {n_real_forecast} test (example, node, horizon) value(s)") + else: + forecast_precip_test = np.zeros((len(X_test), combined["n_nodes"], len(HORIZONS)), dtype=np.float32) + forecast_missing_test = np.ones((len(X_test), combined["n_nodes"], len(HORIZONS)), dtype=np.float32) + print("No real forecast archive found -- evaluating without forecast precipitation") + + x_shared_static_std, _, shared_static_flagged = standardize_static_features( + combined["x_shared_static"], shared_static_cols, + ) + x_gauge_static_std, _, gauge_static_flagged = standardize_static_features( + combined["x_gauge_static"], gauge_static_cols, + ) + print(f"Static feature standardization: shared missingness flags for {shared_static_flagged or 'none'}, " + f"gauge missingness flags for {gauge_static_flagged or 'none'}") + x_shared_static_t = torch.tensor(x_shared_static_std, dtype=torch.float32) + x_gauge_static_t = torch.tensor(x_gauge_static_std, dtype=torch.float32) + is_gauged_t = torch.tensor(combined["is_gauged"], dtype=torch.bool) + edge_index_t = torch.tensor(combined["edge_index"], dtype=torch.long) + edge_attr_t = torch.tensor(combined["edge_attr"], dtype=torch.float32) + basin_id_t = torch.tensor(combined["basin_id"], dtype=torch.long) + horizons_t = torch.tensor(HORIZONS, dtype=torch.float32) + + model = SpatiotemporalGNN( + shared_static_dim=x_shared_static_t.shape[1], gauge_static_dim=x_gauge_static_t.shape[1], + gauge_dynamic_dim=X.shape[-1], edge_dim=edge_attr_t.shape[1], + n_basins=len(BASIN_FILE_NAMES), n_targets=2, quantiles=QUANTILES, + ) + model.load_state_dict(torch.load(model_path, map_location="cpu")) + model.eval() + print(f"Loaded trained model from {model_path}") + + if args.flag_quantile not in QUANTILES: + print(f"--flag-quantile {args.flag_quantile} isn't one of the model's real predicted quantiles " + f"{QUANTILES} -- pick one of those.") + return + default_quantile_idx = QUANTILES.index(args.flag_quantile) + + # Per-horizon quantile_idx map -- every horizon defaults to + # --flag-quantile's index, then real per-horizon overrides replace + # specific entries. Built as an explicit {horizon: idx} dict rather + # than a single shared value, since real testing in this project + # showed the right operating point genuinely differs by horizon + # (see --flag-quantile-overrides' own help text for the real numbers). + quantile_idx_by_horizon = {h: default_quantile_idx for h in HORIZONS} + if args.flag_quantile_overrides: + for pair in args.flag_quantile_overrides.split(","): + pair = pair.strip() + if not pair: + continue + try: + h_str, q_str = pair.split("=") + h_override, q_override = int(h_str), float(q_str) + except ValueError: + print(f"Couldn't parse override '{pair}' -- expected format 'horizon=quantile', e.g. '5=0.9'.") + return + if h_override not in HORIZONS: + print(f"Override horizon {h_override} isn't one of this model's real horizons {HORIZONS}.") + return + if q_override not in QUANTILES: + print(f"Override quantile {q_override} (for horizon {h_override}) isn't one of the " + f"model's real predicted quantiles {QUANTILES} -- pick one of those.") + return + quantile_idx_by_horizon[h_override] = QUANTILES.index(q_override) + + print(f"Flagging a real event when the model's predicted quantile exceeds that station's real " + f"threshold -- not the median, since flood risk is a tail question. Per-horizon quantile: " + f"{ {h: QUANTILES[idx] for h, idx in quantile_idx_by_horizon.items()} }") + + # Confusion matrix per horizon: {horizon: {"tp":.., "fp":.., "fn":.., "tn":..}} + confusion = {h: {"tp": 0, "fp": 0, "fn": 0, "tn": 0} for h in HORIZONS} + fp_precip_values: List[float] = [] + tn_precip_values: List[float] = [] + # Real (0.95 - median) discharge spread, per horizon, across every + # real example and node regardless of tp/fp/fn/tn classification -- + # unlike the precipitation check, this doesn't depend on the + # flagging outcome, so every real prediction contributes. + spread_by_horizon: Dict[int, List[float]] = {h: [] for h in HORIZONS} + if args.check_quantile_spread: + if 0.5 not in QUANTILES or 0.95 not in QUANTILES: + print("--check-quantile-spread needs both 0.5 and 0.95 in this model's real predicted " + f"quantiles {QUANTILES} -- neither is missing normally, but checked defensively.") + args.check_quantile_spread = False + else: + median_idx, q95_idx = QUANTILES.index(0.5), QUANTILES.index(0.95) + + requested_stations: List[str] = [] + station_timeseries_data: Dict[str, Dict[str, list]] = {} + if args.plot_stations: + if 0.5 not in QUANTILES or 0.95 not in QUANTILES: + print("--plot-stations needs both 0.5 and 0.95 in this model's real predicted quantiles " + f"{QUANTILES} -- neither is missing normally, but checked defensively.") + elif args.plot_stations_horizon not in HORIZONS: + print(f"--plot-stations-horizon {args.plot_stations_horizon} isn't one of this model's " + f"real horizons {HORIZONS}.") + else: + requested_stations = [s.strip() for s in args.plot_stations.split(",") if s.strip()] + unknown = [s for s in requested_stations if s not in station_codes] + if unknown: + print(f"Station(s) {unknown} aren't real station codes in this run -- dropping them.") + requested_stations = [s for s in requested_stations if s in station_codes] + median_idx_ts, q95_idx_ts = QUANTILES.index(0.5), QUANTILES.index(0.95) + plot_horizon_idx = HORIZONS.index(args.plot_stations_horizon) + station_timeseries_data = { + code: {"dates": [], "observed": [], "median": [], "q95": []} for code in requested_stations + } + + # {horizon: {"fp_with_forecast": n, "fp_without_forecast": n, + # "all_with_forecast": n, "all_without_forecast": n}} -- + # only meaningful for horizons in FORECAST_LEAD_TIMES, which is + # itself a real, direct subset of HORIZONS this project actually + # has real forecast coverage for. + forecast_fp_stats: Dict[int, Dict[str, int]] = { + h: {"fp_with_forecast": 0, "fp_without_forecast": 0, "all_with_forecast": 0, "all_without_forecast": 0} + for h in HORIZONS if h in FORECAST_LEAD_TIMES + } + fp_details: List[dict] = [] + if args.dump_fp_details is not None and args.dump_fp_details not in HORIZONS: + print(f"--dump-fp-details {args.dump_fp_details} isn't one of this model's real horizons {HORIZONS}.") + return + + with torch.no_grad(): + for i in range(len(X_test)): + x_dynamic_seq = torch.tensor(X_test[i], dtype=torch.float32) + anchor = anchor_dates_test[i] + forecast_precip_i = torch.tensor(forecast_precip_test[i], dtype=torch.float32) + forecast_missing_i = torch.tensor(forecast_missing_test[i], dtype=torch.float32) + pred = model(x_shared_static_t, x_gauge_static_t, x_dynamic_seq, is_gauged_t, + edge_index_t, edge_attr_t, basin_id_t, horizons_t, + forecast_precip_i, forecast_missing_i) + # [n_nodes, n_horizons, n_quantiles] -- every predicted + # quantile, not just one: which specific quantile counts as + # "the flood flag" now genuinely varies by horizon (see + # quantile_idx_by_horizon), so the selection has to happen + # per-horizon below, not once for the whole tensor. + Q_pred_all_quantiles_standardized = pred[:, :, 0, :].numpy() + Q_pred_all_quantiles_real = Q_pred_all_quantiles_standardized * discharge_std + discharge_mean + + if requested_stations: + # Real observed value at this specific (station, target + # date) -- Y_test is standardized the same way discharge + # predictions are, so the same real mean/std un- + # standardizes it correctly. NaN where no real + # observation exists, which plot_station_timeseries + # renders as a genuine gap, not a fabricated value. + target_date_ts = anchor + pd.Timedelta(days=args.plot_stations_horizon - 1) + Y_real_this_horizon = Y_test[i][:, plot_horizon_idx] * discharge_std + discharge_mean + for code in requested_stations: + node_idx_ts = station_codes.index(code) + station_timeseries_data[code]["dates"].append(target_date_ts) + station_timeseries_data[code]["observed"].append(float(Y_real_this_horizon[node_idx_ts])) + station_timeseries_data[code]["median"].append( + float(Q_pred_all_quantiles_real[node_idx_ts, plot_horizon_idx, median_idx_ts])) + station_timeseries_data[code]["q95"].append( + float(Q_pred_all_quantiles_real[node_idx_ts, plot_horizon_idx, q95_idx_ts])) + + if args.check_quantile_spread: + # Real, un-standardized spread -- computed once per + # example here (all horizons, all nodes), not inside the + # per-horizon flagging loop below, since this doesn't + # depend on any station threshold or event label at all. + spread_real = Q_pred_all_quantiles_real[:, :, q95_idx] - Q_pred_all_quantiles_real[:, :, median_idx] + for h_idx, h in enumerate(HORIZONS): + spread_by_horizon[h].extend(spread_real[:, h_idx].tolist()) + + if args.check_fp_precipitation and has_real_precip: + # Real recent precipitation, per node, for THIS example's + # own input window -- one value per node, shared across + # every horizon below (the input window itself doesn't + # change with horizon, only the target date does), same + # ROUTING_HORIZON_COUNT-day real window water_balance_loss + # itself uses during training, not an arbitrarily chosen one. + recent_window = X_test[i][-ROUTING_HORIZON_COUNT:, :, precip_value_idx] # [days, n_nodes], standardized + # Missing days were filled with 0.0 (standardized) at + # window-build time -- treating that as a real 0mm + # reading would be wrong (0 standardized un-standardizes + # to precip_mean, not to a real zero), but for a + # diagnostic sum specifically, filled-missing days + # contribute precip_mean each, a real, if imperfect, + # neutral fill rather than silently skewing the sum -- an + # acceptable simplification for a diagnostic, not + # something that would be acceptable inside the loss + # itself. + recent_precip_sum_real_per_node = recent_window.sum(axis=0) * precip_std + ROUTING_HORIZON_COUNT * precip_mean + + for h_idx, h in enumerate(HORIZONS): + target_date = anchor + pd.Timedelta(days=h - 1) + quantile_idx = quantile_idx_by_horizon[h] + Q_pred_real_this_horizon = Q_pred_all_quantiles_real[:, h_idx, quantile_idx] + track_forecast_fp = args.check_forecast_fp_correlation and h in FORECAST_LEAD_TIMES + for node_idx, station_code in enumerate(station_codes): + if station_code not in station_thresholds: + continue # no real threshold for this station at all -- nothing to evaluate + + threshold = station_thresholds[station_code] + predicted_flag = Q_pred_real_this_horizon[node_idx] >= threshold + is_event = is_real_event_day(station_code, target_date, event_lookup) is not None + if track_forecast_fp: + has_real_forecast = forecast_missing_test[i][node_idx, h_idx] == 0.0 + + if is_event and predicted_flag: + confusion[h]["tp"] += 1 + elif is_event and not predicted_flag: + confusion[h]["fn"] += 1 + elif not is_event and predicted_flag: + confusion[h]["fp"] += 1 + if args.check_fp_precipitation and has_real_precip: + fp_precip_values.append(float(recent_precip_sum_real_per_node[node_idx])) + if track_forecast_fp: + key = "fp_with_forecast" if has_real_forecast else "fp_without_forecast" + forecast_fp_stats[h][key] += 1 + if args.dump_fp_details == h: + fp_details.append({ + "station_code": station_code, + "target_date": target_date.date().isoformat(), + "predicted_value": float(Q_pred_real_this_horizon[node_idx]), + "threshold": float(threshold), + "margin": float(Q_pred_real_this_horizon[node_idx] - threshold), + "days_to_nearest_real_event": days_to_nearest_real_event( + station_code, target_date, event_lookup, + ), + }) + else: + confusion[h]["tn"] += 1 + if args.check_fp_precipitation and has_real_precip: + tn_precip_values.append(float(recent_precip_sum_real_per_node[node_idx])) + + if track_forecast_fp: + key = "all_with_forecast" if has_real_forecast else "all_without_forecast" + forecast_fp_stats[h][key] += 1 + + if requested_stations: + train_end_ts_ts = pd.Timestamp(args.train_end) + train_date_mask_ts = np.array([d <= train_end_ts_ts for d in dates]) + print("\n" + "=" * 70) + print(f"Real station time-series plots, horizon={args.plot_stations_horizon}d") + print("=" * 70) + for code in requested_stations: + node_idx_ts = station_codes.index(code) + # Real, per-station naive baseline -- that station's own + # training-period empirical median/0.95 (real, un- + # standardized L/s), not the single global naive baseline + # used elsewhere for the overall skill percentage -- see + # plot_station_timeseries's own docstring for why a station- + # specific value is the more meaningful comparison here. + real_train_values = (target_tensor[node_idx_ts, train_date_mask_ts] * discharge_std + + discharge_mean) + real_train_values = real_train_values[~np.isnan(real_train_values)] + if len(real_train_values) == 0: + print(f"Station {code}: no real training-period observations at all -- skipping.") + continue + naive_median = float(np.quantile(real_train_values, 0.5)) + naive_q95 = float(np.quantile(real_train_values, 0.95)) + threshold = station_thresholds.get(code, float("nan")) + + plot_path = args.data_root / f"station_timeseries_{code}_h{args.plot_stations_horizon}.png" + plot_station_timeseries( + station_timeseries_data[code]["dates"], station_timeseries_data[code]["observed"], + station_timeseries_data[code]["median"], station_timeseries_data[code]["q95"], + naive_median, naive_q95, threshold, code, args.plot_stations_horizon, plot_path, + ) + print(f"Station {code}: saved to {plot_path} (naive median={naive_median:.1f}, " + f"naive 0.95={naive_q95:.1f}, real threshold={threshold:.1f})") + + if args.check_quantile_spread: + print("\n" + "=" * 70) + print("Real (0.95 - median) discharge quantile spread by horizon (real L/s)") + print("=" * 70) + fp_counts_by_horizon = {h: confusion[h]["fp"] for h in HORIZONS} + for h in HORIZONS: + values = spread_by_horizon[h] + if not values: + continue + arr = np.array(values) + print(f"horizon={h:>3}d: mean spread={arr.mean():>10.2f}, median spread={np.median(arr):>10.2f}, " + f"real fp count={fp_counts_by_horizon[h]:>5}") + # A real, direct check of the actual hypothesis: does the + # spread's own growth track where false positives explode + # (day 5 -> day 10 in the real run that motivated this check), + # rather than just eyeballing two printed columns. + short_horizons = [h for h in HORIZONS if h <= 5 and spread_by_horizon[h]] + long_horizons = [h for h in HORIZONS if h > 5 and spread_by_horizon[h]] + if short_horizons and long_horizons: + short_mean_spread = np.mean([np.mean(spread_by_horizon[h]) for h in short_horizons]) + long_mean_spread = np.mean([np.mean(spread_by_horizon[h]) for h in long_horizons]) + ratio = long_mean_spread / short_mean_spread if short_mean_spread > 0 else float("inf") + print(f"\nMean spread, horizons <=5d: {short_mean_spread:.2f}; horizons >5d: {long_mean_spread:.2f} " + f"(ratio: {ratio:.2f}x)") + if ratio > 2.0: + print(f"The real quantile spread grows substantially ({ratio:.2f}x) beyond day 5 -- " + f"consistent with (not proof of) genuine hedging/calibration widening being the " + f"real driver of the false-positive explosion at longer horizons, rather than a " + f"precipitation-specific mechanism.") + else: + print(f"The real quantile spread does NOT grow substantially beyond day 5 ({ratio:.2f}x) " + f"-- this specific hypothesis isn't well supported either; the real explanation " + f"for the false-positive jump likely needs further investigation.") + + if args.plot_single_trace: + try: + station_code_req, anchor_str = args.plot_single_trace.split("=") + except ValueError: + print(f"--plot-single-trace '{args.plot_single_trace}' isn't in the expected " + f"'station_code=anchor_date' format.") + return + if station_code_req not in station_codes: + print(f"Station {station_code_req} isn't one of this run's real station codes.") + return + node_idx_req = station_codes.index(station_code_req) + anchor_req = pd.Timestamp(anchor_str) + matches = [i for i, a in enumerate(anchor_dates_test) if pd.Timestamp(a) == anchor_req] + if not matches: + print(f"Anchor date {anchor_str} isn't one of this run's real test anchor dates.") + return + i = matches[0] + + with torch.no_grad(): + x_dynamic_seq = torch.tensor(X_test[i], dtype=torch.float32) + forecast_precip_i = torch.tensor(forecast_precip_test[i], dtype=torch.float32) + forecast_missing_i = torch.tensor(forecast_missing_test[i], dtype=torch.float32) + pred = model(x_shared_static_t, x_gauge_static_t, x_dynamic_seq, is_gauged_t, + edge_index_t, edge_attr_t, basin_id_t, horizons_t, + forecast_precip_i, forecast_missing_i) + quantile_values_standardized = pred[node_idx_req, :, 0, :].numpy() # [n_horizons, n_quantiles] + quantile_values_real = quantile_values_standardized * discharge_std + discharge_mean + + threshold = station_thresholds.get(station_code_req, float("nan")) + trace_plot_path = args.data_root / f"single_trace_{station_code_req}_{anchor_str}.png" + plot_single_trace(HORIZONS, QUANTILES, quantile_values_real, threshold, + ROUTING_HORIZON_COUNT, station_code_req, anchor_str, trace_plot_path) + print(f"\nSaved single-trace plot to {trace_plot_path}") + print(f"Real predicted values (real L/s), station={station_code_req}, anchor={anchor_str}:") + for h_idx, h in enumerate(HORIZONS): + row = ", ".join(f"q{q}={quantile_values_real[h_idx, q_idx]:.1f}" for q_idx, q in enumerate(QUANTILES)) + print(f" horizon={h:>3}d: {row}") + + if args.dump_fp_details is not None: + h = args.dump_fp_details + print("\n" + "=" * 70) + print(f"Real false positive details at horizon={h}d (n={len(fp_details)})") + print("=" * 70) + if not fp_details: + print("No real false positives at this horizon -- nothing to dump.") + else: + fp_df = pd.DataFrame(fp_details) + output_path = args.dump_fp_output or (args.data_root / f"fp_details_horizon_{h}.csv") + fp_df.to_csv(output_path, index=False) + print(f"Saved to {output_path}") + + print("\nReal false positive count by station:") + print(fp_df["station_code"].value_counts().to_string()) + + n_near_miss = int((fp_df["days_to_nearest_real_event"] <= 3).sum()) + n_no_event_at_all = int(np.isinf(fp_df["days_to_nearest_real_event"]).sum()) + print(f"\n{n_near_miss}/{len(fp_df)} real false positives are within 3 real days of an actual " + f"real event (near-misses, not random noise); {n_no_event_at_all}/{len(fp_df)} are at " + f"stations with no real event anywhere in the real record at all.") + print(f"\nReal date range of these false positives: " + f"{fp_df['target_date'].min()} to {fp_df['target_date'].max()}") + print(f"Real margin (predicted - threshold) stats: mean={fp_df['margin'].mean():.2f}, " + f"median={fp_df['margin'].median():.2f}, max={fp_df['margin'].max():.2f}") + + if args.check_forecast_fp_correlation: + print("\n" + "=" * 70) + print("Real forecast availability: false positives vs. the real base rate, by horizon") + print("=" * 70) + if not forecast_fp_stats: + print(f"No requested horizons fall in FORECAST_LEAD_TIMES {FORECAST_LEAD_TIMES} -- nothing to check.") + for h, stats in forecast_fp_stats.items(): + n_fp = stats["fp_with_forecast"] + stats["fp_without_forecast"] + n_all = stats["all_with_forecast"] + stats["all_without_forecast"] + if n_fp == 0 or n_all == 0: + print(f"horizon={h:>2}d: no real false positives or no real examples at all -- nothing to compare.") + continue + fp_forecast_rate = stats["fp_with_forecast"] / n_fp + base_forecast_rate = stats["all_with_forecast"] / n_all + print(f"horizon={h:>2}d: {fp_forecast_rate*100:.1f}% of real false positives had real forecast " + f"data available, vs. {base_forecast_rate*100:.1f}% base rate across all real examples " + f"(n_fp={n_fp}, n_all={n_all})") + # A real, direct verdict, not just printed numbers to eyeball -- + # flags horizons where false positives are genuinely + # disproportionately drawn from forecast-available examples, + # not just nominally different by a small amount. + flagged_horizons = [] + for h, stats in forecast_fp_stats.items(): + n_fp = stats["fp_with_forecast"] + stats["fp_without_forecast"] + n_all = stats["all_with_forecast"] + stats["all_without_forecast"] + if n_fp == 0 or n_all == 0: + continue + fp_rate = stats["fp_with_forecast"] / n_fp + base_rate = stats["all_with_forecast"] / n_all + if base_rate > 0 and fp_rate / base_rate > 1.5: + flagged_horizons.append(h) + if flagged_horizons: + print(f"\nReal false positives are disproportionately drawn from forecast-available examples " + f"at horizon(s) {flagged_horizons} -- real, direct evidence the forecast-conditioning " + f"pathway itself is contributing to false positives there, not generic horizon-driven " + f"hedging alone.") + else: + print(f"\nNo real horizon shows false positives disproportionately drawn from forecast-" + f"available examples -- this specific hypothesis isn't well supported by this " + f"real evidence either.") + + if args.check_fp_precipitation: + if not has_real_precip: + print("\n--check-fp-precipitation requested, but no real precipitation channel is present " + "in this run -- nothing to check.") + elif not fp_precip_values or not tn_precip_values: + print("\n--check-fp-precipitation requested, but there were no real false positives or no " + "real true negatives to compare -- nothing to check.") + else: + fp_arr, tn_arr = np.array(fp_precip_values), np.array(tn_precip_values) + print("\n" + "=" * 70) + print(f"Real recent precipitation (mm, summed over the last {ROUTING_HORIZON_COUNT} real " + f"days), false positives vs. true negatives") + print("=" * 70) + print(f"False positives (n={len(fp_arr)}): mean={fp_arr.mean():.2f}mm, median={np.median(fp_arr):.2f}mm") + print(f"True negatives (n={len(tn_arr)}): mean={tn_arr.mean():.2f}mm, median={np.median(tn_arr):.2f}mm") + if fp_arr.mean() > tn_arr.mean(): + print(f"False positives show real, higher recent precipitation on average " + f"({fp_arr.mean():.2f}mm vs {tn_arr.mean():.2f}mm) -- consistent with (not proof " + f"of) water_balance_loss's ET=0/no-storage gap pushing the model to over-predict " + f"discharge whenever real precipitation is high.") + else: + print(f"False positives do NOT show higher recent precipitation than true negatives -- " + f"this specific hypothesis isn't supported by this real evidence; the real " + f"explanation for weak precision likely lies elsewhere.") + + print("\n" + "=" * 70) + print("Flood-detection confusion matrix by lead time (real held-out test data)") + print("=" * 70) + rows = [] + for h in HORIZONS: + c = confusion[h] + precision = c["tp"] / (c["tp"] + c["fp"]) if (c["tp"] + c["fp"]) > 0 else float("nan") + recall = c["tp"] / (c["tp"] + c["fn"]) if (c["tp"] + c["fn"]) > 0 else float("nan") + rows.append({"horizon_days": h, "tp": c["tp"], "fp": c["fp"], "fn": c["fn"], "tn": c["tn"], + "precision": precision, "recall": recall}) + result_df = pd.DataFrame(rows) + print(result_df.to_string(index=False)) + + output_path = args.data_root / "flood_detection_evaluation.csv" + result_df.to_csv(output_path, index=False) + print(f"\nSaved to {output_path}") + + try: + confusion_plot_path = args.data_root / "flood_detection_confusion_matrices.png" + plot_confusion_matrices_by_horizon(confusion, HORIZONS, confusion_plot_path) + print(f"Saved confusion matrix grid to {confusion_plot_path}") + + pr_plot_path = args.data_root / "flood_detection_precision_recall.png" + plot_precision_recall_by_horizon(result_df, pr_plot_path) + print(f"Saved precision/recall-vs-horizon plot to {pr_plot_path}") + except Exception as e: + print(f" [warning] failed to save evaluation plots: {e}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/evaluation/identify_flood_events.py b/scripts/evaluation/identify_flood_events.py new file mode 100644 index 0000000000000000000000000000000000000000..a87a41819d632ffbc1a02ecdbf7d58cccea616d2 --- /dev/null +++ b/scripts/evaluation/identify_flood_events.py @@ -0,0 +1,151 @@ +""" +Identifies real historical high-flow events from discharge and +water-level records, station by station. This is the ground truth +against which any flood-prediction claim eventually has to be checked +-- "beats a naive mean-baseline" and "would have flagged the real +floods that actually happened" are different questions, and only the +second one matters for this project's actual purpose. + +Threshold: each station's own historical 95th/99th percentile, not a +shared absolute value across stations. These rivers have very +different scales (median discharge ranges from under 1,000 to over +9,000 L/s across real stations in this project), so a fixed absolute +threshold would flag "high flow" at a small tributary while missing +real extremes at a larger one. A station-relative threshold puts every +station's real flood-scale events on the same footing. + +Events, not days: a real flood spans several consecutive days above +threshold, and counting each day as a separate "event" would both +inflate the count and obscure how long real high-flow periods actually +last -- duration matters for anything downstream (evacuation windows, +infrastructure impact), not just peak magnitude. + +Usage: + python -m scripts.identify_flood_events --data-root datasets +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +try: + from src.graph.dynamic_features import build_discharge_timeseries, build_waterlevel_timeseries +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_discharge_timeseries, build_waterlevel_timeseries + + +def find_events(series: pd.Series, threshold: float, min_duration_days: int = 1) -> pd.DataFrame: + """ + Groups consecutive above-threshold days into discrete events for a + single station's real time series. + + A gap of even one day below threshold ends the current event -- + real recession rarely means the flow dropped and immediately rose + again, so treating a brief dip as "still the same event" would + understate how variable real hydrographs actually are. + + Returns one row per event: start date, end date, duration, peak + value, peak date. Events shorter than min_duration_days are + dropped -- a single anomalous reading (a possible data error, or a + genuinely brief spike) is a different thing from a sustained high- + flow event, and conflating them would dilute what "event" means for + later evaluation. + """ + above = series >= threshold + if not above.any(): + return pd.DataFrame(columns=["start_date", "end_date", "duration_days", "peak_value", "peak_date"]) + + # Assign a group id that increments each time the series drops + # below threshold -- consecutive True values share a group. + group_id = (~above).cumsum() + events = [] + for gid, group in series[above].groupby(group_id[above]): + duration = len(group) + if duration < min_duration_days: + continue + events.append({ + "start_date": group.index.min(), + "end_date": group.index.max(), + "duration_days": duration, + "peak_value": group.max(), + "peak_date": group.idxmax(), + }) + return pd.DataFrame(events) + + +def identify_station_events( + wide: pd.DataFrame, percentile: float = 95.0, min_duration_days: int = 1, +) -> pd.DataFrame: + """ + Runs find_events for every station in a [date x station_code] wide + table, using each station's own real historical percentile as its + threshold. Stations with too little real data to compute a + meaningful percentile (fewer than 30 real observations, an + arbitrary but reasonable floor -- a percentile from a handful of + readings isn't a real threshold) are skipped, not given a + threshold computed from noise. + """ + all_events = [] + for station_code in wide.columns: + series = wide[station_code].dropna() + if len(series) < 30: + continue + threshold = np.percentile(series, percentile) + station_events = find_events(series, threshold, min_duration_days) + if station_events.empty: + continue + station_events["station_code"] = station_code + station_events["threshold"] = threshold + station_events["threshold_percentile"] = percentile + all_events.append(station_events) + + if not all_events: + return pd.DataFrame(columns=["station_code", "start_date", "end_date", "duration_days", + "peak_value", "peak_date", "threshold", "threshold_percentile"]) + return pd.concat(all_events, ignore_index=True).sort_values(["station_code", "start_date"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Identify real historical high-flow events") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None) + parser.add_argument("--start-date", type=str, default="2013-01-01") + parser.add_argument("--end-date", type=str, default="2026-12-31") + parser.add_argument("--percentile", type=float, default=95.0, + help="Station-relative percentile defining the threshold for a high-flow day.") + parser.add_argument("--min-duration-days", type=int, default=1) + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + + date_range = (args.start_date, args.end_date) + output_dir = args.output_dir or args.data_root + + discharge_wide = build_discharge_timeseries(nodes_df, args.data_root / "hydrometric", date_range) + discharge_events = identify_station_events(discharge_wide, args.percentile, args.min_duration_days) + print(f"Discharge: {len(discharge_events)} real event(s) across " + f"{discharge_events['station_code'].nunique() if not discharge_events.empty else 0} station(s)") + if not discharge_events.empty: + print(discharge_events.to_string(index=False)) + discharge_out = output_dir / "flood_events_discharge.csv" + discharge_events.to_csv(discharge_out, index=False) + print(f"Saved to {discharge_out}\n") + + waterlevel_wide = build_waterlevel_timeseries(nodes_df, args.data_root / "hydrometric", date_range) + waterlevel_events = identify_station_events(waterlevel_wide, args.percentile, args.min_duration_days) + print(f"Water level: {len(waterlevel_events)} real event(s) across " + f"{waterlevel_events['station_code'].nunique() if not waterlevel_events.empty else 0} station(s)") + if not waterlevel_events.empty: + print(waterlevel_events.to_string(index=False)) + waterlevel_out = output_dir / "flood_events_waterlevel.csv" + waterlevel_events.to_csv(waterlevel_out, index=False) + print(f"Saved to {waterlevel_out}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/__pycache__/enrich_reach_graph.cpython-311.pyc b/scripts/graph_building/__pycache__/enrich_reach_graph.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..efe152112452a4c5e8e0db32133e3dbeb898d3af Binary files /dev/null and b/scripts/graph_building/__pycache__/enrich_reach_graph.cpython-311.pyc differ diff --git a/scripts/graph_building/build_dynamic_tensors.py b/scripts/graph_building/build_dynamic_tensors.py new file mode 100644 index 0000000000000000000000000000000000000000..b33764493a182dd0029ae698ff7a7ead6bff6dee --- /dev/null +++ b/scripts/graph_building/build_dynamic_tensors.py @@ -0,0 +1,117 @@ +""" +Builds genuine [n_nodes, T] dynamic tensors (discharge, groundwater, +climate) for a basin's reach graph and feeds them directly into +physics_losses.py's routing_consistency_loss -- the actual integration +point dynamic_features.py exists for. Without this script, that +function has real inputs it could consume but nothing actually +producing them. + +Usage: + python -m scripts.build_dynamic_tensors --data-root datasets --basin risle +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +try: + from src.graph.dynamic_features import ( + build_discharge_timeseries, build_groundwater_timeseries, + build_climate_timeseries, assemble_dynamic_tensor, + ) + from src.graph.physics_losses import build_routing_index, routing_consistency_loss +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import ( + build_discharge_timeseries, build_groundwater_timeseries, + build_climate_timeseries, assemble_dynamic_tensor, + ) + from src.graph.physics_losses import build_routing_index, routing_consistency_loss + +BASIN_FILE_NAMES = {0: "eure", 1: "risle"} + + +def run_for_basin(data_root: Path, basin_id: int, file_key: str, date_range, skip_climate: bool) -> None: + graph_dir = data_root / "reach_graph" + nodes_path = graph_dir / f"{file_key}_nodes_enriched.csv" + edges_path = graph_dir / f"{file_key}_edges.csv" + if not nodes_path.exists() or not edges_path.exists(): + print(f"{file_key}: missing enriched nodes/edges -- run build_reach_graphs.py + " + f"enrich_reach_graph.py first. Skipping.") + return + + nodes_df = pd.read_csv(nodes_path) + edges_df = pd.read_csv(edges_path) + print(f"--- {file_key}: {len(nodes_df)} nodes, {len(edges_df)} edges ---") + + discharge_wide = build_discharge_timeseries(nodes_df, data_root / "hydrometric", date_range) + Q, dates = assemble_dynamic_tensor(nodes_df, discharge_wide) + n_real = int((~np.isnan(Q)).sum()) + print(f"discharge tensor: {Q.shape}, {n_real}/{Q.size} real (non-NaN) values " + f"({100*n_real/Q.size:.3f}% coverage -- expect this to be tiny, only real " + f"gauges with real observations ever have a value here)") + + level_wide, depth_wide = build_groundwater_timeseries(nodes_df, data_root / "ades", date_range) + level_tensor, _ = assemble_dynamic_tensor(nodes_df, level_wide) + depth_tensor, _ = assemble_dynamic_tensor(nodes_df, depth_wide) + print(f"groundwater level tensor: {level_tensor.shape}, " + f"{int((~np.isnan(level_tensor)).sum())} real values") + + climate_tensors = {} + if not skip_climate and (data_root / "safran").exists(): + try: + climate_dict = build_climate_timeseries(nodes_df, data_root / "safran", date_range) + for var, wide in climate_dict.items(): + tensor, _ = assemble_dynamic_tensor(nodes_df, wide) + climate_tensors[var] = tensor + print(f"climate variables: {list(climate_tensors.keys())}") + except Exception as e: + print(f"climate skipped (error: {e})") + + out_dir = graph_dir / "dynamic" + out_dir.mkdir(exist_ok=True) + save_kwargs = { + "discharge": Q, "groundwater_level": level_tensor, "groundwater_depth": depth_tensor, + "dates": np.array([str(d) for d in dates]), "station_codes": nodes_df["station_code"].values, + } + save_kwargs.update({f"climate_{k}": v for k, v in climate_tensors.items()}) + out_path = out_dir / f"{file_key}_dynamic.npz" + np.savez(out_path, **save_kwargs) + print(f"saved to {out_path}") + + # The actual integration: routing_consistency_loss needs Q + routing_index together. + routing_index = build_routing_index(nodes_df, edges_df, timestep_hours=24.0) + print(f"routing_index: {len(routing_index)} edge(s) with a usable lag at this timestep") + + loss = routing_consistency_loss(Q, routing_index) + print(f"routing_consistency_loss on the REAL discharge tensor: {loss}") + if np.isnan(loss): + print(" ^ NaN. This is exactly the thing worth checking before assuming this loss " + "is usable: real Q is almost entirely NaN (only real gauges with real " + "observations have values), and if routing_consistency_loss doesn't mask " + "NaN out of its residuals before averaging, one missing value anywhere " + "poisons the entire loss to NaN. See whether this fired.") + print() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build dynamic tensors and wire into physics_losses.py") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--basin", choices=["eure", "risle", "both"], default="both") + parser.add_argument("--start-date", type=str, default="2013-01-01") + parser.add_argument("--end-date", type=str, default="2026-12-31") + parser.add_argument("--skip-climate", action="store_true") + args = parser.parse_args() + + date_range = (args.start_date, args.end_date) + basins = BASIN_FILE_NAMES.items() if args.basin == "both" else \ + [(k, v) for k, v in BASIN_FILE_NAMES.items() if v == args.basin] + + for basin_id, file_key in basins: + run_for_basin(args.data_root, basin_id, file_key, date_range, args.skip_climate) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/build_reach_graphs.py b/scripts/graph_building/build_reach_graphs.py new file mode 100644 index 0000000000000000000000000000000000000000..63ccbfb8b1baaa5c4d5150bc0f3b1dc7410b9ca8 --- /dev/null +++ b/scripts/graph_building/build_reach_graphs.py @@ -0,0 +1,188 @@ +""" +Build the reach-based river network graph for both basins from real BD TOPO +data, and report on every step so problems are visible immediately instead +of surfacing later as a silent wrong answer downstream. + +Usage: + python -m scripts.build_reach_graphs --data-root datasets +""" +import argparse +import json +import sys +from pathlib import Path + +import pandas as pd + +try: + from src.graph.build_reach_graph import ( + check_flow_direction_coverage, load_troncons_for_basin, build_node_link_digraph, + best_component_for_stations, snap_gauges_to_reach_graph, insert_virtual_nodes, + build_reach_graph_tables, summarize_confluences_by_tributary, TRIBUTARY_NAMES, normalize_toponym, + ) + from src.data.loaders.station_elevations import StationElevationsLoader + from src.data.river_graph import assign_basin_id +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.build_reach_graph import ( + check_flow_direction_coverage, load_troncons_for_basin, build_node_link_digraph, + best_component_for_stations, snap_gauges_to_reach_graph, insert_virtual_nodes, + build_reach_graph_tables, summarize_confluences_by_tributary, TRIBUTARY_NAMES, normalize_toponym, + ) + from src.data.loaders.station_elevations import StationElevationsLoader + from src.data.river_graph import assign_basin_id + +BASIN_NAMES = {0: "La Eure", 1: "La Risle"} + + +def run_for_basin( + troncon_geojson: dict, + stations_df: pd.DataFrame, + basin_id: int, + virtual_node_spacing_km: float, + anchor_radius_km: float, + component_search_radius_km: float, +) -> "tuple[pd.DataFrame, pd.DataFrame] | None": + name = BASIN_NAMES[basin_id] + print("=" * 70) + print(f"{name}") + print("=" * 70) + + basin_stations = stations_df[stations_df["basin_id"] == basin_id].copy() + + matched = load_troncons_for_basin( + troncon_geojson, basin_id=basin_id, + anchor_stations=basin_stations, max_distance_from_anchor_km=anchor_radius_km, + ) + print(f"Matched {len(matched)} tronçons by name AND within {anchor_radius_km} km of a " + f"real gauge (name-only matching pulled in geographically unrelated tronçons " + f"in earlier runs -- see build_reach_graph.py's load_troncons_for_basin docstring)") + if not matched: + print(" Nothing matched -- try a larger --anchor-radius-km, or check " + "TRIBUTARY_NAMES against this file's actual toponym values.") + return None + + print() + print("Flow direction field coverage (sens_de_l_ecoulement), all matched tronçons:") + coverage = check_flow_direction_coverage(matched) + for value, count in sorted(coverage.items(), key=lambda x: -x[1]): + print(f" {value!r}: {count}") + print(" If this doesn't show recognizable direction values, " + "build_node_link_digraph's sens_downstream_values/sens_upstream_values " + "need updating before its elevation fallback is the only thing working.") + + print() + G, report = build_node_link_digraph(matched) + print(report) + + n_known_tributaries = len(TRIBUTARY_NAMES.get(basin_id, [])) - 1 # exclude the main river itself + print(f" ({n_known_tributaries} known named tributaries for this basin -- confluence count " + f"should be in this neighborhood, not far beyond it, if a tributary joins the main " + f"river roughly once)") + tributary_summary = summarize_confluences_by_tributary(G) + if not tributary_summary.empty: + counts = tributary_summary["joining_river"].value_counts() + print(f" Confluences by joining river (post name-normalization): {dict(counts)}") + if len(counts) > n_known_tributaries + 3: # some slack for real sub-tributary branching + print(f" NOTE: more distinct joining-river names ({len(counts)}) than known " + f"tributaries ({n_known_tributaries}) -- worth checking whether some of " + f"these are actually the same river under yet another name variant " + f"normalize_toponym doesn't catch, or genuinely unlisted tributaries.") + + if report.disconnected_components > 1: + print(f" {report.disconnected_components} component(s) found. Selecting by which " + f"one actually contains our real gauges, not by raw size -- the two aren't " + f"guaranteed to agree, and in earlier runs they didn't.") + G, gauge_counts = best_component_for_stations(G, basin_stations, search_radius_km=component_search_radius_km) + if len(gauge_counts) > 1: + sorted_counts = sorted(gauge_counts.values(), reverse=True) + print(f" Gauges captured per component: {sorted_counts[:5]}{'...' if len(sorted_counts) > 5 else ''}") + if sorted_counts[0] < len(basin_stations): + print(f" WARNING: the winning component only captures {sorted_counts[0]} of " + f"{len(basin_stations)} gauges -- this basin's real network may still be " + f"fragmented across more than one component even after the anchor filter.") + + print() + print(f"Snapping {len(basin_stations)} real gauge(s) onto the reach graph...") + snapped = snap_gauges_to_reach_graph(G, basin_stations) + print(snapped[["station_code", "reach_u", "reach_v", "reach_fraction", "snap_distance_km"]] + .sort_values("snap_distance_km", ascending=False).to_string(index=False)) + unsnapped = snapped["reach_u"].isna().sum() + if unsnapped: + print(f" WARNING: {unsnapped} station(s) got no nearest edge at all -- " + f"they're likely outside this basin's matched tronçon set.") + large_snap = (snapped["snap_distance_km"] > 2.0).sum() + if large_snap: + print(f" NOTE: {large_snap} station(s) snapped more than 2 km from any " + f"reach -- worth checking those aren't actually on an unmatched tributary.") + + print() + G_virtual, n_inserted = insert_virtual_nodes(G, spacing_km=virtual_node_spacing_km) + print(f"Inserted {n_inserted} virtual node(s) at ~{virtual_node_spacing_km} km spacing " + f"({G.number_of_nodes()} real nodes -> {G_virtual.number_of_nodes()} total)") + + print() + nodes_df, edges_df = build_reach_graph_tables(G_virtual, snapped, basin_id=basin_id) + n_confluences = int(nodes_df["is_confluence"].sum()) + print(f"Final table: {len(nodes_df)} nodes ({nodes_df['is_gauged'].sum()} gauged, " + f"{n_confluences} confluences, {len(nodes_df) - nodes_df['is_gauged'].sum() - n_confluences} " + f"virtual/other), {len(edges_df)} edges") + + return nodes_df, edges_df + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build reach-based river network graphs") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--virtual-node-spacing-km", type=float, default=5.0) + parser.add_argument("--anchor-radius-km", type=float, default=20.0, + help="Max distance from a real gauge for a name-matched tronçon to be kept") + parser.add_argument("--component-search-radius-km", type=float, default=5.0, + help="How close a gauge must be to a component to count as 'in' it") + parser.add_argument("--output-dir", type=Path, default=None, + help="Defaults to /reach_graph") + args = parser.parse_args() + + troncon_path = args.data_root / "bdtopo_hydro" / "troncon_hydrographique.geojson" + if not troncon_path.exists(): + print(f"Missing {troncon_path} -- run scripts/download_bdtopo_hydro.py first.") + sys.exit(1) + troncon_geojson = json.loads(troncon_path.read_text()) + + stations_df = StationElevationsLoader(data_path=args.data_root / "station_elevations.csv").load() + stations_df["basin_id"] = stations_df["station_code"].apply(assign_basin_id) + unmatched = stations_df["basin_id"].isna().sum() + if unmatched: + print(f"NOTE: {unmatched} station(s) matched no basin prefix and are excluded entirely.\n") + stations_df = stations_df.dropna(subset=["basin_id"]) + stations_df["basin_id"] = stations_df["basin_id"].astype(int) + + output_dir = args.output_dir or (args.data_root / "reach_graph") + output_dir.mkdir(parents=True, exist_ok=True) + + all_nodes, all_edges = [], [] + for basin_id, file_key in [(0, "eure"), (1, "risle")]: + result = run_for_basin(troncon_geojson, stations_df, basin_id, args.virtual_node_spacing_km, + args.anchor_radius_km, args.component_search_radius_km) + print() + if result is None: + continue + nodes_df, edges_df = result + nodes_df.to_csv(output_dir / f"{file_key}_nodes.csv", index=False) + edges_df.to_csv(output_dir / f"{file_key}_edges.csv", index=False) + print(f"Saved {file_key}_nodes.csv / {file_key}_edges.csv to {output_dir}") + print() + all_nodes.append(nodes_df) + all_edges.append(edges_df) + + if all_nodes: + print("=" * 70) + print("SUMMARY") + print("=" * 70) + combined_nodes = pd.concat(all_nodes, ignore_index=True) + combined_edges = pd.concat(all_edges, ignore_index=True) + print(f"Total: {len(combined_nodes)} nodes, {len(combined_edges)} edges across both basins") + print(f"Gauged: {combined_nodes['is_gauged'].sum()}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/compute_cumulative_catchment.py b/scripts/graph_building/compute_cumulative_catchment.py new file mode 100644 index 0000000000000000000000000000000000000000..605ca81595245e98acc25c59050323f0848a92ef --- /dev/null +++ b/scripts/graph_building/compute_cumulative_catchment.py @@ -0,0 +1,208 @@ +""" +Fills catchment_area_km2 for every reach graph node (confluences, +splits, rejoins, virtual infill -- not just the ~8 real gauges Hub'Eau +publishes it for), using BD TOPO's incremental catchment polygons +(bassin_versant_topographique.geojson). + +ALGORITHM: + 1. Spatially assign each incremental polygon to the edges it contains + (an edge's midpoint falling inside a polygon means that edge's + reach is the one that polygon's incremental drainage feeds). + 2. For any node, cumulative catchment area = sum of the AREAS OF + DISTINCT POLYGONS among every edge upstream of it (walked via the + real directed graph, unbounded -- unlike the split/rejoin check's + 15km search radius, catchment area needs the true full upstream + drainage, however far that extends). "Distinct" matters: many + edges (and the virtual nodes on them) share the same polygon, since + BD TOPO's confluence-to-confluence stretches are far coarser than + our node resolution -- summing per-edge would massively overcount. + +CAVEAT: the polygon assignment step needs geopandas, which isn't +available in the environment this was developed in (no network access +to install it). The graph-traversal/area-summation logic (the actual +novel part) is tested with synthetic polygon assignments standing in +for a real spatial join -- but the spatial join itself has not been run +against real data. Please verify the polygon assignment step +specifically (e.g. spot-check a few edges against the map) before +trusting the final areas. + +Usage: + python -m scripts.compute_cumulative_catchment --data-root datasets +""" +import argparse +import sys +from pathlib import Path + +import networkx as nx +import pandas as pd + + +def assign_edges_to_polygons(edges_df: pd.DataFrame, nodes_df: pd.DataFrame, polygons_path: Path) -> pd.Series: + """ + Spatial join: for each edge, which incremental catchment polygon + contains its midpoint. Returns a Series indexed like edges_df, with + the polygon's unique identifier (or NaN if no polygon contains it -- + can happen near basin edges or bbox boundaries). + + NEEDS geopandas -- not verified in the environment this was written + in, see module docstring. + """ + import geopandas as gpd + from shapely.geometry import Point + + polygons = gpd.read_file(polygons_path) + if "cleabs" in polygons.columns: + poly_id_col = "cleabs" + else: + polygons = polygons.reset_index().rename(columns={"index": "_poly_idx"}) + poly_id_col = "_poly_idx" + + coord = nodes_df.set_index("station_code")[["latitude", "longitude"]] + midpoints = [] + for _, e in edges_df.iterrows(): + try: + src, tgt = coord.loc[e["source"]], coord.loc[e["target"]] + except KeyError: + midpoints.append(None) + continue + midpoints.append(Point((src["longitude"] + tgt["longitude"]) / 2, + (src["latitude"] + tgt["latitude"]) / 2)) + + edge_points = gpd.GeoDataFrame({"edge_idx": range(len(edges_df))}, geometry=midpoints, crs=polygons.crs) + joined = gpd.sjoin(edge_points.dropna(subset=["geometry"]), polygons[[poly_id_col, "geometry"]], + how="left", predicate="within") + joined = joined.drop_duplicates("edge_idx") # a midpoint exactly on a shared boundary could match >1 polygon + + result = pd.Series(index=range(len(edges_df)), dtype=object) + result.loc[joined["edge_idx"]] = joined[poly_id_col].values + return result + + +def compute_cumulative_catchment( + nodes_df: pd.DataFrame, + edges_df: pd.DataFrame, + edge_polygon_id: pd.Series, + polygon_areas: dict, +) -> pd.Series: + """ + For every node, sum the areas of distinct polygons among all edges + upstream of it (inclusive of the edge(s) arriving at the node + itself). Pure graph-traversal + arithmetic -- no geospatial + dependency, fully testable without geopandas (see the test in this + script's accompanying verification, or run this module's functions + directly against synthetic data). + + Args: + nodes_df: reach graph nodes [station_code, ...] + edges_df: reach graph edges [source, target, ...], same row + order/index as edge_polygon_id. + edge_polygon_id: from assign_edges_to_polygons (or an equivalent + manually-built mapping for testing) -- polygon ID per edge, + aligned to edges_df's index. + polygon_areas: {polygon_id: area_km2} + + Returns: + Series indexed like nodes_df, cumulative catchment area in km2 + (NaN for a node with no upstream edges assigned to any polygon + at all -- a true headwater with no BD TOPO catchment coverage, + distinct from zero). + """ + G = nx.DiGraph() + G.add_nodes_from(nodes_df["station_code"]) + edge_to_poly = {} + for i, e in edges_df.iterrows(): + G.add_edge(e["source"], e["target"]) + poly = edge_polygon_id.get(i) + if pd.notna(poly): + edge_to_poly[(e["source"], e["target"])] = poly + + results = {} + for node in nodes_df["station_code"]: + ancestors = nx.ancestors(G, node) | {node} + # every edge whose TARGET is in this node's ancestor-or-self set + # is an edge that feeds into this node's upstream drainage + polys_here = set() + for (u, v), poly in edge_to_poly.items(): + if v in ancestors: + polys_here.add(poly) + if not polys_here: + results[node] = float("nan") + else: + results[node] = sum(polygon_areas.get(p, 0.0) for p in polys_here) + + return pd.Series(results).reindex(nodes_df["station_code"]) + + +def run_for_basin(data_root: Path, basin_id: int, file_key: str) -> None: + graph_dir = data_root / "reach_graph" + enriched_path = graph_dir / f"{file_key}_nodes_enriched.csv" + edges_path = graph_dir / f"{file_key}_edges.csv" + polygons_path = data_root / "bdtopo_hydro" / "bassin_versant_topographique.geojson" + + if not enriched_path.exists() or not edges_path.exists(): + print(f"{file_key}: missing enriched nodes/edges, run build_reach_graphs.py + " + f"enrich_reach_graph.py first. Skipping.") + return + if not polygons_path.exists(): + print(f"{file_key}: no {polygons_path} found -- run download_bdtopo_hydro.py first. Skipping.") + return + + nodes_df = pd.read_csv(enriched_path) + edges_df = pd.read_csv(edges_path) + print(f"{file_key}: {len(nodes_df)} nodes, {len(edges_df)} edges") + + try: + import geopandas as gpd + except ImportError: + print(f"{file_key}: geopandas not installed -- cannot do the spatial join. Skipping.") + return + + polygons = gpd.read_file(polygons_path) + poly_id_col = "cleabs" if "cleabs" in polygons.columns else "_poly_idx" + if poly_id_col == "_poly_idx": + polygons = polygons.reset_index().rename(columns={"index": poly_id_col}) + polygons_metric = polygons.to_crs(epsg=2154) + polygon_areas = dict(zip(polygons[poly_id_col], polygons_metric.geometry.area / 1e6)) + + edge_polygon_id = assign_edges_to_polygons(edges_df, nodes_df, polygons_path) + n_assigned = edge_polygon_id.notna().sum() + print(f" {n_assigned}/{len(edges_df)} edges assigned to a polygon " + f"({len(edges_df) - n_assigned} unassigned -- near bbox/basin edges, expected)") + + cumulative = compute_cumulative_catchment(nodes_df, edges_df, edge_polygon_id, polygon_areas) + nodes_df["cumulative_catchment_area_km2"] = cumulative.values + + n_covered = nodes_df["cumulative_catchment_area_km2"].notna().sum() + print(f" cumulative_catchment_area_km2 coverage: {n_covered}/{len(nodes_df)} nodes " + f"(vs. {nodes_df['catchment_area_km2'].notna().sum()} from Hub'Eau's own reported values)") + + # Cross-check against Hub'Eau's real values where both exist, as a + # sanity check on the whole pipeline -- they measure different things + # (Hub'Eau: official gauge catchment; this: BD TOPO polygon sum) so + # exact agreement isn't expected, but they should be in the same + # ballpark for a real gauge, not off by an order of magnitude. + both = nodes_df.dropna(subset=["catchment_area_km2", "cumulative_catchment_area_km2"]) + if not both.empty: + both = both.copy() + both["ratio"] = both["cumulative_catchment_area_km2"] / both["catchment_area_km2"] + print(f" Cross-check vs Hub'Eau (real gauges only): ratio mean={both['ratio'].mean():.2f}, " + f"range=[{both['ratio'].min():.2f}, {both['ratio'].max():.2f}]") + print(both[["station_code", "catchment_area_km2", "cumulative_catchment_area_km2", "ratio"]] + .to_string(index=False)) + + nodes_df.to_csv(enriched_path, index=False) + print(f" updated {enriched_path}") + print() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compute cumulative catchment area from BD TOPO polygons") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + args = parser.parse_args() + + for basin_id, file_key in [(0, "eure"), (1, "risle")]: + run_for_basin(args.data_root, basin_id, file_key) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/compute_edge_width.py b/scripts/graph_building/compute_edge_width.py new file mode 100644 index 0000000000000000000000000000000000000000..58d2edf75f033eef1aa795e31e5be99238843f7b --- /dev/null +++ b/scripts/graph_building/compute_edge_width.py @@ -0,0 +1,109 @@ +""" +Derives approximate channel width per real tronçon (river reach +segment) from BD TOPO data already downloaded in this project -- +surface_hydrographique.geojson (real water-surface polygons) and +troncon_hydrographique.geojson (real centerline geometry) -- no new +data source needed. + +width_m = surface_polygon_area_m2 / troncon_length_m + +This is real, derivable channel geometry, not a full cross-sectional +profile -- true cross-section (a real depth profile across the +channel) needs bathymetric/LIDAR survey data, which hasn't been +sourced or verified for this project. Width alone is still a +physically meaningful hydraulic signal: it's the dominant term in +Manning's-equation-style conveyance estimates, and the reason a wide +shallow reach and a narrow deep one carry very different discharge for +the same water level -- exactly the kind of physics a plain +distance/elevation-drop edge attribute can't express on its own. + +Spatial join: each tronçon is matched to whichever real surface +polygon its centerline geometry falls within (or, failing that, the +nearest one within a small tolerance) -- a tronçon with no matching +surface polygon at all gets width_m = NaN, not a fabricated default, +since "no real surface polygon found" is meaningfully different from +"the channel is zero-width." + +Usage: + python -m scripts.compute_edge_width --data-root datasets +""" +import argparse +from pathlib import Path + +import geopandas as gpd +import pandas as pd + + +def compute_widths(troncons: gpd.GeoDataFrame, surfaces: gpd.GeoDataFrame) -> pd.DataFrame: + """ + Real spatial join: for each tronçon, find the surface polygon(s) it + intersects, sum their real area (a tronçon can legitimately span + more than one mapped surface polygon), divide by real tronçon + length. Returns one row per tronçon with a real cleabs identifier + and width_m (NaN where no real surface polygon matched). + """ + troncons = troncons.copy() + troncons["length_m"] = troncons.geometry.length + + joined = gpd.sjoin(troncons[["cleabs", "length_m", "geometry"]], surfaces[["geometry"]], + how="left", predicate="intersects") + + area_by_troncon = surfaces.geometry.area + joined["surface_area_m2"] = joined["index_right"].map( + lambda idx: area_by_troncon.loc[idx] if pd.notna(idx) and idx in area_by_troncon.index else float("nan") + ) + + result = joined.groupby("cleabs").agg( + length_m=("length_m", "first"), + total_surface_area_m2=("surface_area_m2", lambda x: x.sum() if x.notna().any() else float("nan")), + ).reset_index() + + result["width_m"] = result["total_surface_area_m2"] / result["length_m"] + n_matched = result["width_m"].notna().sum() + print(f"{n_matched}/{len(result)} real tronçon(s) matched to a real surface polygon " + f"({len(result) - n_matched} have no real surface data -- width_m will be NaN for those, not fabricated)") + + return result[["cleabs", "length_m", "width_m"]] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Derive real channel width from BD TOPO surface polygons") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args() + + bdtopo_dir = args.data_root / "bdtopo_hydro" + troncons_path = bdtopo_dir / "troncon_hydrographique.geojson" + surfaces_path = bdtopo_dir / "surface_hydrographique.geojson" + + if not troncons_path.exists() or not surfaces_path.exists(): + print(f"Missing {troncons_path} or {surfaces_path} -- run download_bdtopo_hydro.py first.") + return + + print(f"Loading real tronçons from {troncons_path}...") + troncons = gpd.read_file(troncons_path) + print(f"Loading real surface polygons from {surfaces_path}...") + surfaces = gpd.read_file(surfaces_path) + print(f"{len(troncons)} real tronçon(s), {len(surfaces)} real surface polygon(s)") + + if troncons.crs != surfaces.crs: + print(f"CRS mismatch ({troncons.crs} vs {surfaces.crs}) -- reprojecting surfaces to match tronçons") + surfaces = surfaces.to_crs(troncons.crs) + + widths = compute_widths(troncons, surfaces) + + real_widths = widths["width_m"].dropna() + if len(real_widths) > 0: + print(f"\nReal width distribution: min={real_widths.min():.1f}m, median={real_widths.median():.1f}m, " + f"max={real_widths.max():.1f}m") + print("Sanity-check these against what you know of the real rivers before trusting them -- " + "an implausible max (e.g. hundreds of meters) usually means a tronçon matched an " + "unrelated, much larger surface polygon nearby, not a real channel width.") + + output_path = args.output or (args.data_root / "edge_widths.csv") + widths.to_csv(output_path, index=False) + print(f"\nSaved to {output_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/diagnose_confluences.py b/scripts/graph_building/diagnose_confluences.py new file mode 100644 index 0000000000000000000000000000000000000000..d910d648908ecb877fe0a14cfbfd65dc7d988c71 --- /dev/null +++ b/scripts/graph_building/diagnose_confluences.py @@ -0,0 +1,135 @@ +""" +Diagnose the confluence count in a saved reach graph: real tributary +junctions, or a data artifact (e.g. the same physical reach digitized +twice, or a tile-boundary duplication in BD TOPO). + +Works two ways: + 1. Against your EXISTING risle_nodes.csv/risle_edges.csv (no rerun + needed) -- checks whether each flagged confluence's incoming edges + have suspiciously similar distance_km/elevation_drop_m, which is + the signature of two near-identical tronçons rather than two + genuinely different ones. + 2. Conclusively, once you rerun scripts/build_reach_graphs.py with the + cleabs fix -- if two incoming edges at the same node share the same + cleabs (tronçon ID), that's definitive: same tronçon counted twice, + not two different ones. + +Usage: + python -m scripts.diagnose_confluences --data-root datasets --basin risle +""" +import argparse +from pathlib import Path + +import pandas as pd + + +def diagnose(nodes_path: Path, edges_path: Path, n_samples: int = 10) -> None: + nodes_df = pd.read_csv(nodes_path) + edges_df = pd.read_csv(edges_path) + + has_is_confluence = "is_confluence" in nodes_df.columns + has_toponym = "toponym" in edges_df.columns + has_cleabs = "cleabs" in edges_df.columns + + if has_is_confluence: + confluence_codes = set(nodes_df[nodes_df["is_confluence"]]["station_code"]) + confluences = edges_df[edges_df["target"].isin(confluence_codes)]["target"].value_counts() + raw_in_degree = edges_df["target"].value_counts() + raw_confluences = raw_in_degree[raw_in_degree >= 2] + print(f"{len(confluence_codes)} REAL confluence(s) -- different named river, independent " + f"upstream source, not a split that rejoins downstream -- out of " + f"{len(raw_confluences)} node(s) with raw in-degree >= 2 " + f"({len(nodes_df)} total nodes, {len(edges_df)} edges)") + print(f" The gap ({len(raw_confluences) - len(confluence_codes)} nodes) is same-river " + f"segmentation and/or braided channels that split and rejoin -- both excluded " + f"from the count above, using the authoritative is_confluence column computed " + f"with full graph topology (this script can't re-derive the split-rejoin check " + f"from the flat CSV alone, which is exactly why that column exists).") + elif has_toponym: + name_counts = edges_df.dropna(subset=["toponym"]).groupby("target")["toponym"].nunique() + confluences = name_counts[name_counts > 1] + raw_in_degree = edges_df["target"].value_counts() + raw_confluences = raw_in_degree[raw_in_degree >= 2] + print(f"{len(confluences)} REAL confluence(s) (different named river actually joining), " + f"out of {len(raw_confluences)} node(s) with raw in-degree >= 2 " + f"({len(nodes_df)} total nodes, {len(edges_df)} edges)") + print(f" The gap ({len(raw_confluences) - len(confluences)} nodes) is same-river fine " + f"tronçon segmentation, not real branching -- this is now excluded from the count.") + print(" NOTE: this file predates the split-rejoin fix (no is_confluence column) -- a " + "braided channel with differently-named branches that rejoin downstream would " + "still be miscounted here as a real confluence. Re-run " + "scripts/build_reach_graphs.py for the corrected count.") + else: + in_degree = edges_df["target"].value_counts() + confluences = in_degree[in_degree >= 2] + print(f"{len(confluences)} node(s) with in-degree >= 2, out of {len(nodes_df)} total nodes " + f"and {len(edges_df)} edges") + print() + print("NOTE: this edges.csv doesn't have a 'toponym' column -- it was generated before " + "the name-based confluence fix. Every number above uses the old, less reliable " + "in-degree >= 2 rule. Re-run scripts/build_reach_graphs.py for a corrected count " + "and for this script to give a definitive per-node answer instead of a proxy.") + print() + + sample = confluences.head(n_samples) + n_confirmed_duplicate = 0 + n_suspicious = 0 + + for node in sample.index: + incoming = edges_df[edges_df["target"] == node] + cols = ["source", "target", "distance_km", "elevation_drop_m"] + cols += [c for c in ("toponym", "cleabs") if c in incoming.columns] + print(f"Node {node}: {len(incoming)} incoming edge(s)") + print(incoming[cols].to_string(index=False)) + + if has_toponym: + n_names = incoming["toponym"].nunique() + if n_names > 1: + print(f" -> REAL: {n_names} distinct named river(s) join here.") + else: + print(f" -> Same toponym on every incoming edge -- not counted as a real " + f"confluence (this shouldn't appear in the sample above; if it does, " + f"something's inconsistent between this script and build_reach_graph.py).") + elif has_cleabs and incoming["cleabs"].notna().all(): + if incoming["cleabs"].nunique() < len(incoming): + print(" -> CONFIRMED DUPLICATE: two+ incoming edges share the same cleabs " + "(same physical tronçon counted more than once).") + n_confirmed_duplicate += 1 + else: + dists = incoming["distance_km"].dropna() + if len(dists) >= 2 and (dists.max() - dists.min()) < 0.05: + print(" -> SUSPICIOUS: incoming edges have near-identical distance_km " + "(within 50m) -- consistent with the same reach being counted twice, " + "not two distinct tributaries joining.") + n_suspicious += 1 + print() + + print("=" * 60) + if has_toponym: + print(f"All {len(sample)} sampled nodes have >1 distinct named river joining -- " + f"that's the definition being used, so this should always be 100%. If any " + f"printed 'Same toponym' above, report it as a bug.") + elif has_cleabs: + print(f"Of {len(sample)} sampled confluences: {n_confirmed_duplicate} confirmed " + f"duplicate-tronçon artifacts, {len(sample) - n_confirmed_duplicate} have " + f"genuinely distinct incoming tronçons (real confluence candidates).") + else: + print(f"Of {len(sample)} sampled confluences: {n_suspicious} look suspicious " + f"(near-identical incoming edge distances). Re-run scripts/build_reach_graphs.py " + f"for a conclusive, name-based answer instead of this proxy signal.") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Diagnose reach-graph confluence counts") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--basin", choices=["eure", "risle"], required=True) + parser.add_argument("--n-samples", type=int, default=10) + args = parser.parse_args() + + graph_dir = args.data_root / "reach_graph" + diagnose(graph_dir / f"{args.basin}_nodes.csv", graph_dir / f"{args.basin}_edges.csv", + n_samples=args.n_samples) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/diagnose_persistence_baseline.py b/scripts/graph_building/diagnose_persistence_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..15fd9725a511af06d72a3ac1e7f2976fbbd823ed --- /dev/null +++ b/scripts/graph_building/diagnose_persistence_baseline.py @@ -0,0 +1,124 @@ +""" +Diagnostic: adds a naive persistence baseline (tomorrow's anomaly = +today's real anomaly, no model at all) to the climatology comparison, +computed directly from real anomaly data -- no trained model needed. + +This exists specifically to distinguish two very different explanations +for the anomaly model losing to climatology at short horizons: + (a) there's genuinely no learnable short-term signal beyond + climatology for that variable, or + (b) the model/training setup has a real problem, and even the + simplest possible non-climatology guess (yesterday's value) + would have beaten climatology if given the chance. + +If persistence beats climatology at h=1 but the trained model doesn't, +that's a training/architecture issue, not a genuine predictability +limit -- a very different, much more actionable finding. + +Usage: + python -m scripts.diagnose_persistence_baseline --data-root datasets \ + --train-end 2023-12-31 --test-end 2026-12-31 +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +try: + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies + +DEFAULT_HORIZONS = [1, 5, 10, 30, 60, 90, 180] + + +def evaluate_persistence(anomalies: pd.DataFrame, horizons: list) -> pd.DataFrame: + """ + For every real (date, cell) anomaly value, compares: + - persistence_rmse: predicting that SAME day's anomaly repeated + forward h days (i.e. "nothing has changed since today") + - climatology_only_rmse: predicting zero anomaly (pure climatology) + against the REAL anomaly that actually occurred h days later. + Both computed only where the real target h days ahead exists. + """ + values = anomalies.values + n_dates, n_cells = values.shape + rows = [] + + for h in horizons: + if h >= n_dates: + continue + today = values[:n_dates - h, :] + future = values[h:, :] + valid = ~np.isnan(today) & ~np.isnan(future) + if not valid.any(): + continue + persistence_pred = today[valid] # "tomorrow = today" + actual = future[valid] + persistence_rmse = np.sqrt(np.mean((persistence_pred - actual) ** 2)) + climatology_rmse = np.sqrt(np.mean(actual ** 2)) # predicting anomaly = 0 + rows.append({ + "horizon_days": h, "n_pairs": int(valid.sum()), + "persistence_rmse": persistence_rmse, "climatology_only_rmse": climatology_rmse, + "persistence_beats_climatology": persistence_rmse < climatology_rmse, + }) + return pd.DataFrame(rows) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Persistence-vs-climatology diagnostic") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None) + parser.add_argument("--bbox-margin-deg", type=float, default=0.3) + parser.add_argument("--train-start", type=str, default="2013-01-01") + parser.add_argument("--train-end", type=str, default="2023-12-31") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--horizons", type=int, nargs="+", default=DEFAULT_HORIZONS) + parser.add_argument("--n-harmonics", type=int, default=2) + parser.add_argument("--min-months-covered", type=int, default=8) + args = parser.parse_args() + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + m = args.bbox_margin_deg + bbox = (nodes_df["longitude"].min() - m, nodes_df["latitude"].min() - m, + nodes_df["longitude"].max() + m, nodes_df["latitude"].max() + m) + + print("Loading real ERA5 grid data...") + climate_dict, grid_coords = build_climate_grid_timeseries( + args.data_root / "safran", bbox, (args.train_start, args.test_end) + ) + print(f"Loaded {len(climate_dict)} real variable(s) across {len(grid_coords)} real grid cell(s)\n") + + train_start, train_end, test_end = ( + pd.Timestamp(args.train_start), pd.Timestamp(args.train_end), pd.Timestamp(args.test_end) + ) + + print("=" * 80) + print(f"{'Variable':<15}{'Horizon':>8} {'Persistence RMSE':>17} {'Climatology RMSE':>17} {'Persistence wins?':>18}") + print("=" * 80) + + for var, wide in climate_dict.items(): + train_wide = wide[(wide.index >= train_start) & (wide.index <= train_end)] + test_wide = wide[(wide.index > train_end) & (wide.index <= test_end)] + + coeffs = fit_harmonic_climatology(train_wide, n_harmonics=args.n_harmonics, + min_months_covered=args.min_months_covered) + if not coeffs: + continue + test_anomalies = compute_anomalies(test_wide, coeffs, args.n_harmonics) + result = evaluate_persistence(test_anomalies, args.horizons) + + for _, row in result.iterrows(): + print(f"{var:<15}{int(row['horizon_days']):>8} {row['persistence_rmse']:>17.4f} " + f"{row['climatology_only_rmse']:>17.4f} {str(row['persistence_beats_climatology']):>18}") + print() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/graph_building/enrich_reach_graph.py b/scripts/graph_building/enrich_reach_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..16dee195e7bf30a87bef51463c5b49b3b681fefb --- /dev/null +++ b/scripts/graph_building/enrich_reach_graph.py @@ -0,0 +1,126 @@ +""" +Enriches the reach graph's node tables (from scripts/build_reach_graphs.py) +with every available feature loader, via node_features.py's base_nodes_df +path -- closing the gap where the reach graph and the feature pipeline +existed separately but nothing actually connected them. + +Usage: + python -m scripts.enrich_reach_graph --data-root datasets +""" +import argparse +import sys +from pathlib import Path + +import pandas as pd + +try: + from src.graph.node_features import build_node_features +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.node_features import build_node_features + +BASIN_FILE_NAMES = {0: "eure", 1: "risle"} + + +def enrich_basin( + data_root: Path, basin_id: int, skip_climate: bool = False, + date_range: "tuple[str, str] | None" = None, +) -> None: + file_key = BASIN_FILE_NAMES[basin_id] + graph_dir = data_root / "reach_graph" + nodes_path = graph_dir / f"{file_key}_nodes.csv" + + if not nodes_path.exists(): + print(f"{file_key}: no {nodes_path} found -- run scripts/build_reach_graphs.py first. Skipping.") + return + + base_nodes = pd.read_csv(nodes_path) + print(f"{file_key}: {len(base_nodes)} nodes loaded from {nodes_path}") + + enriched, report = build_node_features( + base_nodes_df=base_nodes, + idpr_path=data_root / "idpr.csv", + ades_path=data_root / "ades", + safran_path=None if skip_climate else data_root / "safran", + # --skip-climate exists specifically because add_safran_features has + # not been verified at reach-graph scale (~2,900 nodes) and is the + # prime suspect for a real hang there, unlike everything else in + # this pipeline which has been checked at this scale already (see + # add_groundwater_features's rewrite, which hit exactly this class + # of bug once before). Use this flag to get unblocked on everything + # else while that gets diagnosed with the real source. + catchment_path=data_root / "catchment_area.csv", + hydrometric_path=data_root / "hydrometric", + bdcavites_path=data_root / "bdcavites" / "cavite_localisee.geojson", + # Real, confirmed bug fix: this was never passed at all before, + # despite add_cavites_features and its real KD-tree/haversine + # logic being fully built and already generic over base_nodes_df + # -- the reach graph's ~4,500 nodes never got a real cavity + # feature purely because this one call site never wired it in, + # not because of any real gauge-only restriction in the feature + # code itself. Defaults to the "naturelle"-only filter inside + # add_cavites_features -- confirmed via scripts/analysis/ + # check_cavity_proximity.py as the real, defensible choice + # (an unfiltered check found 24/27 real gauges within 1km of + # *some* cavity, dominated by quarries/civil structures with no + # hydrological meaning; "naturelle" is what actually indicates + # real karst connectivity). + date_range=date_range, + # Safe to include here, unlike an earlier version of this script + # assumed: target_* attaches via a left-merge on station_code, + # and real gauge codes ("H...") can never collide with BD TOPO + # hydrographic node IDs ("NOEUDHYD...") or virtual infill node + # IDs ("VIRTUAL::..."), confirmed against the real station code + # namespace -- so only real gauge rows ever get a real target + # value; every confluence/virtual node correctly gets NaN, not + # a mislabeled one. + ) + print(report) + + n_climate = enriched["climate_precip_mm"].notna().sum() if "climate_precip_mm" in enriched.columns else 0 + n_catchment = enriched["catchment_area_km2"].notna().sum() if "catchment_area_km2" in enriched.columns else 0 + n_targets = enriched["target_discharge_m3s_mean"].notna().sum() if "target_discharge_m3s_mean" in enriched.columns else 0 + n_cavites = enriched["distance_to_nearest_cavity_km"].notna().sum() if "distance_to_nearest_cavity_km" in enriched.columns else 0 + print(f" climate coverage: {n_climate}/{len(enriched)} nodes") + print(f" target (discharge) coverage: {n_targets}/{len(enriched)} nodes " + f"(expected: only real gauges -- {int(base_nodes['is_gauged'].sum())} in this basin)") + print(f" catchment_area coverage: {n_catchment}/{len(enriched)} nodes " + f"(expected: only real gauges have one -- see node_features.py's " + f"add_catchment_features docstring; this is what water_balance_loss " + f"can actually use right now, everything else needs a cumulative " + f"catchment estimate this script doesn't build)") + print(f" cavity proximity coverage: {n_cavites}/{len(enriched)} nodes " + f"(graph-wide, naturelle-only filter -- see add_cavites_features)") + + out_path = graph_dir / f"{file_key}_nodes_enriched.csv" + enriched.to_csv(out_path, index=False) + print(f" saved to {out_path}") + print() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Enrich reach graph nodes with feature loaders") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--skip-climate", action="store_true", + help="Skip SAFRAN/ERA5 climate features -- use if that step hangs at reach-graph scale") + parser.add_argument("--start-date", type=str, default="2013-01-01", + help="Start of the training period (inclusive). Default 2013-01-01: the window " + "maximizing discharge-gauge coverage (6/8 stations, 13,084 observations) -- " + "see the date-range analysis this default came from before changing it.") + parser.add_argument("--end-date", type=str, default="2026-12-31", + help="End of the training period (inclusive).") + parser.add_argument("--no-date-filter", action="store_true", + help="Disable date filtering entirely -- each source uses its own full history") + args = parser.parse_args() + + date_range = None if args.no_date_filter else (args.start_date, args.end_date) + if date_range: + print(f"Using date range: {date_range[0]} to {date_range[1]}") + print() + + for basin_id in BASIN_FILE_NAMES: + enrich_basin(args.data_root, basin_id, skip_climate=args.skip_climate, date_range=date_range) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/tests/__pycache__/cross_validate.cpython-311.pyc b/scripts/tests/__pycache__/cross_validate.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b32aae7ab4c11040260f7ad99e5df51d999f9d29 Binary files /dev/null and b/scripts/tests/__pycache__/cross_validate.cpython-311.pyc differ diff --git a/scripts/tests/__pycache__/test_climate_climatology.cpython-311.pyc b/scripts/tests/__pycache__/test_climate_climatology.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3af039c6df54d618413d0c736cabd8e27b7cb02b Binary files /dev/null and b/scripts/tests/__pycache__/test_climate_climatology.cpython-311.pyc differ diff --git a/scripts/tests/__pycache__/test_climate_climatology_grid.cpython-311.pyc b/scripts/tests/__pycache__/test_climate_climatology_grid.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f826ed35e451ce10ae0a6f3cfe291a81a12c77af Binary files /dev/null and b/scripts/tests/__pycache__/test_climate_climatology_grid.cpython-311.pyc differ diff --git a/scripts/tests/__pycache__/train_climate_anomaly_model.cpython-311.pyc b/scripts/tests/__pycache__/train_climate_anomaly_model.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b682a5e4fe7c0bca42ab1b3f70c1a5787e2de77f Binary files /dev/null and b/scripts/tests/__pycache__/train_climate_anomaly_model.cpython-311.pyc differ diff --git a/scripts/tests/cross_validate.py b/scripts/tests/cross_validate.py new file mode 100644 index 0000000000000000000000000000000000000000..5e496418811eb2efb695e3f864b0c02af71f2db1 --- /dev/null +++ b/scripts/tests/cross_validate.py @@ -0,0 +1,229 @@ +""" +Rolling-window cross-validation for the climate anomaly model -- +built specifically because single train/val/test runs proved unstable +against real data: restricting an otherwise-identical test window from +2024-2026 down to 2024-2025 (removing one year) flipped runoff_mm from +"beats climatology at 5 of 7 horizons" to "beats climatology nowhere at +all", and temp_C from winning at all 7 horizons to losing at 3 of them. +That's not noise to shrug off -- it means no single run's per-horizon +table is trustworthy on its own, and re-running with yet another date +range and reading whichever result looks best is the wrong fix. + +This trains and evaluates across several genuinely independent +(non-overlapping test period) folds with an expanding training window, +then reports, per (variable, horizon), the FRACTION of folds where the +model actually beat climatology and the mean/std of the RMSE margin -- +a "win" that holds up in 4 of 4 folds is real signal; a "win" that +shows up in 1 of 4 is very likely the kind of single-run artifact that +misled the earlier single-window runs. + +Usage: + python -m scripts.cross_validate_climate_anomaly_model --data-root datasets +""" +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import torch + +try: + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies, aggregate_anomalies, REGIME_CONFIG + from scripts.tests.train_climate_anomaly_model import train_one_variable +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies, aggregate_anomalies, REGIME_CONFIG + from scripts.tests.train_climate_anomaly_model import train_one_variable + +# Expanding-window folds, each with a genuinely NON-OVERLAPPING test +# period -- this is what actually lets results from different folds be +# compared as independent evidence, rather than four correlated re-reads +# of mostly the same years. Spans as much of the real 2013-2026 ERA5 +# record as possible while keeping every fold's train/val/test properly +# separated in time (no leakage from a later fold's test period into an +# earlier fold's training set, since every fold's train window starts +# at the same point and only extends forward). +DEFAULT_FOLDS = [ + {"train_start": "2013-01-01", "train_end": "2016-12-31", "val_end": "2018-12-31", "test_end": "2020-12-31"}, + {"train_start": "2013-01-01", "train_end": "2018-12-31", "val_end": "2020-12-31", "test_end": "2022-12-31"}, + {"train_start": "2013-01-01", "train_end": "2020-12-31", "val_end": "2022-12-31", "test_end": "2024-12-31"}, + {"train_start": "2013-01-01", "train_end": "2022-12-31", "val_end": "2024-12-31", "test_end": "2026-12-31"}, +] +# Val periods widened from 1 year to 2 years (24 months) vs the original +# design -- confirmed as a real, not hypothetical, problem against real +# data: the monthly regime (lookback=12 + max horizon=6 = 18 months +# minimum to produce even one training example) got "val examples: 0" +# in every fold under the original 1-year val windows, silently skipping +# that entire regime rather than crashing. 24 months clears the 18-month +# floor with real margin for a stable early-stopping signal, not just +# the bare minimum. Test windows remain genuinely non-overlapping across +# folds (2019-2020, 2021-2022, 2023-2024, 2025-2026), same principle as +# before, just shifted to make room for the wider val period. + + +def run_cross_validation( + climate_dict: dict, variables: list, folds: list, regime_names: list, + max_epochs: int, patience: int, batch_size: int, lr: float, weight_decay: float, + n_harmonics: int, min_months_covered: int, device: str, +) -> pd.DataFrame: + all_rows = [] + + for fold_idx, fold in enumerate(folds): + train_start = pd.Timestamp(fold["train_start"]) + train_end = pd.Timestamp(fold["train_end"]) + val_end = pd.Timestamp(fold["val_end"]) + test_end = pd.Timestamp(fold["test_end"]) + + print("=" * 70) + print(f"FOLD {fold_idx + 1}/{len(folds)}: train [{train_start.date()}, {train_end.date()}], " + f"val (...{val_end.date()}], test (...{test_end.date()}]") + print("=" * 70) + + for var in variables: + if var not in climate_dict: + continue + wide = climate_dict[var] + train_wide = wide[(wide.index >= train_start) & (wide.index <= train_end)] + val_wide = wide[(wide.index > train_end) & (wide.index <= val_end)] + test_wide = wide[(wide.index > val_end) & (wide.index <= test_end)] + + coeffs = fit_harmonic_climatology(train_wide, n_harmonics=n_harmonics, + min_months_covered=min_months_covered) + if not coeffs: + print(f" {var}: insufficient climatology coverage this fold, skipping") + continue + + train_daily = compute_anomalies(train_wide, coeffs, n_harmonics) + val_daily = compute_anomalies(val_wide, coeffs, n_harmonics) + test_daily = compute_anomalies(test_wide, coeffs, n_harmonics) + + # One model per (variable, regime) per fold -- same reasoning + # as train_climate_anomaly_model.py: daily resolution is the + # wrong target past ~2 weeks, so each regime gets its own + # appropriately-aggregated series and its own native horizons. + for regime_name, regime in REGIME_CONFIG.items(): + if regime_name not in regime_names: + continue + if regime["freq"] is None: + train_anomalies, val_anomalies, test_anomalies = train_daily, val_daily, test_daily + else: + train_anomalies = aggregate_anomalies(train_daily, regime["freq"]) + val_anomalies = aggregate_anomalies(val_daily, regime["freq"]) + test_anomalies = aggregate_anomalies(test_daily, regime["freq"]) + + result = train_one_variable( + f"{var}_{regime_name}", train_anomalies, val_anomalies, test_anomalies, + regime["horizons_native"], regime["lookback"], + max_epochs, patience, batch_size, lr, weight_decay, device, + ) + if result["status"] != "trained": + continue + + for row in result["skill_by_horizon"]: + all_rows.append({ + "fold": fold_idx + 1, "variable": var, "regime": regime_name, + "horizon_native": row["horizon_days"], + "model_rmse": row["model_rmse"], "climatology_only_rmse": row["climatology_only_rmse"], + "model_beats_climatology": row["model_beats_climatology"], + "rmse_margin_pct": 100 * (row["climatology_only_rmse"] - row["model_rmse"]) / row["climatology_only_rmse"], + }) + + return pd.DataFrame(all_rows) + + +def summarize_cross_validation(results: pd.DataFrame) -> pd.DataFrame: + """ + The actual answer this whole script exists to produce: per + (variable, regime, horizon), across all folds, what fraction + actually beat climatology, and what the margin looked like on + average -- this is what should be trusted, not any single fold's + own table. Grouped by regime as well as horizon now -- "horizon 3" + means 3 days in the daily regime, 3 weeks in the weekly regime, and + 3 months in the monthly regime, so conflating them would silently + average together numbers that don't refer to the same real horizon. + """ + summary = results.groupby(["variable", "regime", "horizon_native"]).agg( + n_folds=("fold", "count"), + n_folds_beat_climatology=("model_beats_climatology", "sum"), + mean_rmse_margin_pct=("rmse_margin_pct", "mean"), + std_rmse_margin_pct=("rmse_margin_pct", "std"), + ).reset_index() + summary["fraction_folds_beat"] = summary["n_folds_beat_climatology"] / summary["n_folds"] + summary["verdict"] = summary["fraction_folds_beat"].apply( + lambda f: "CONSISTENT WIN" if f == 1.0 else ("CONSISTENT LOSS" if f == 0.0 else "UNSTABLE") + ) + return summary.sort_values(["variable", "regime", "horizon_native"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Rolling-window cross-validation for the climate anomaly model") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None) + parser.add_argument("--bbox-margin-deg", type=float, default=0.3) + parser.add_argument("--regimes", type=str, nargs="+", default=list(REGIME_CONFIG.keys()), + help="Which resolution regimes to cross-validate (default: all).") + parser.add_argument("--n-harmonics", type=int, default=2) + parser.add_argument("--min-months-covered", type=int, default=8) + parser.add_argument("--max-epochs", type=int, default=200) + parser.add_argument("--patience", type=int, default=10) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--weight-decay", type=float, default=1e-4) + parser.add_argument("--variables", type=str, nargs="+", default=None) + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using device: {device}") + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + m = args.bbox_margin_deg + bbox = (nodes_df["longitude"].min() - m, nodes_df["latitude"].min() - m, + nodes_df["longitude"].max() + m, nodes_df["latitude"].max() + m) + + print("Loading real ERA5 grid data once, reused across every fold...") + climate_dict, grid_coords = build_climate_grid_timeseries( + args.data_root / "safran", bbox, ("2013-01-01", "2026-12-31") + ) + print(f"Loaded {len(climate_dict)} real variable(s) across {len(grid_coords)} real grid cell(s)\n") + + variables = args.variables or list(climate_dict.keys()) + output_dir = args.output_dir or (args.data_root / "climate_anomaly_models" / "cross_validation") + output_dir.mkdir(parents=True, exist_ok=True) + + results = run_cross_validation( + climate_dict, variables, DEFAULT_FOLDS, args.regimes, + args.max_epochs, args.patience, args.batch_size, args.lr, args.weight_decay, + args.n_harmonics, args.min_months_covered, device, + ) + + results_path = output_dir / "raw_fold_results.csv" + results.to_csv(results_path, index=False) + print(f"\nSaved raw per-fold results to {results_path}") + + summary = summarize_cross_validation(results) + summary_path = output_dir / "cross_validation_summary.csv" + summary.to_csv(summary_path, index=False) + + print("\n" + "=" * 70) + print("CROSS-VALIDATION SUMMARY -- this is the trustworthy answer, not any single fold") + print("=" * 70) + print(summary.to_string(index=False)) + print(f"\nSaved to {summary_path}") + + print("\nConsistent wins (real, repeatable skill) by variable and regime:") + for var in summary["variable"].unique(): + for regime in summary[summary["variable"] == var]["regime"].unique(): + var_regime_summary = summary[(summary["variable"] == var) & (summary["regime"] == regime)] + consistent = var_regime_summary[var_regime_summary["verdict"] == "CONSISTENT WIN"]["horizon_native"].tolist() + unstable = var_regime_summary[var_regime_summary["verdict"] == "UNSTABLE"]["horizon_native"].tolist() + print(f" {var} / {regime}: consistent wins at {consistent}, unstable (don't trust either way) at {unstable}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/tests/test_climate_climatology.py b/scripts/tests/test_climate_climatology.py new file mode 100644 index 0000000000000000000000000000000000000000..3e554fbd375bca5a33fc80719986aa17d1f53e20 --- /dev/null +++ b/scripts/tests/test_climate_climatology.py @@ -0,0 +1,114 @@ +""" +Two things get real validation here, together: + 1. SAFRANLoader / build_climate_timeseries against real era5_*.nc files + -- flagged as untested every time it's come up in this project (no + real ERA5 files were available in the development environment). + 2. The harmonic climatology model (climate_climatology.py) against + real climate variables specifically -- previously only validated + against real discharge as a stand-in. + +Uses a genuine held-out split (fit on an early period, evaluate on a +later one the model never saw), and the FAIR flat-mean baseline (mean +computed from training data only, not the test period's own true +average -- using the test period's own mean is a look-ahead comparison +no real forecast could replicate; this exact mistake was already caught +and fixed once for discharge, applying the same fix here from the start). + +Usage: + python -m scripts.test_climate_climatology --data-root datasets \ + --train-end 2023-12-31 --test-end 2026-12-31 +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +try: + from src.graph.dynamic_features import build_climate_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, evaluate_climatology_skill +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_climate_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, evaluate_climatology_skill + + +def main() -> None: + parser = argparse.ArgumentParser(description="Test climatology against real ERA5 data") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None, + help="Defaults to /station_elevations.csv") + parser.add_argument("--train-start", type=str, default="2013-01-01") + parser.add_argument("--train-end", type=str, default="2023-12-31") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--n-harmonics", type=int, default=2) + parser.add_argument("--min-months-covered", type=int, default=8) + args = parser.parse_args() + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + print(f"Loaded {len(nodes_df)} real station(s) from {stations_path}") + + safran_path = args.data_root / "safran" + if not safran_path.exists(): + print(f"No {safran_path} found -- point --data-root at the directory containing era5_*.nc files.") + return + + print(f"Loading real ERA5 data from {safran_path} (this may take a while)...") + try: + climate_dict = build_climate_timeseries(nodes_df, safran_path, (args.train_start, args.test_end)) + except Exception as e: + print(f"FAILED to load/build climate timeseries: {e}") + print("This is exactly the failure mode that couldn't be tested before -- please share the " + "full error/traceback so it can be fixed against the real error, not guessed at again.") + raise + + print(f"Loaded {len(climate_dict)} real climate variable(s): {list(climate_dict.keys())}") + print() + + train_start, train_end, test_end = pd.Timestamp(args.train_start), pd.Timestamp(args.train_end), pd.Timestamp(args.test_end) + + for var, wide in climate_dict.items(): + print("=" * 70) + print(f"Variable: {var}") + print("=" * 70) + train = wide[(wide.index >= train_start) & (wide.index <= train_end)] + test = wide[(wide.index > train_end) & (wide.index <= test_end)] + print(f"Train: {train.shape} ({train_start.date()} to {train_end.date()})") + print(f"Test: {test.shape} ({train_end.date()} to {test_end.date()}, genuinely held out)") + + n_real_train = train.notna().sum().sum() + n_real_test = test.notna().sum().sum() + print(f"Real (non-NaN) values -- train: {n_real_train}, test: {n_real_test}") + if n_real_train == 0 or n_real_test == 0: + print("Not enough real data in one of the splits -- skipping this variable.") + print() + continue + + coeffs = fit_harmonic_climatology(train, n_harmonics=args.n_harmonics, + min_months_covered=args.min_months_covered) + n_fit = len(coeffs) + n_skipped = len(wide.columns) - n_fit + print(f"Fitted climatology for {n_fit}/{len(wide.columns)} station(s) " + f"({n_skipped} skipped -- insufficient data or seasonal coverage)") + + if not coeffs: + print("Nothing fit -- skipping evaluation for this variable.") + print() + continue + + train_means = {col: train[col].mean() for col in coeffs.keys()} + skill = evaluate_climatology_skill(test, coeffs, train_means=train_means, n_harmonics=args.n_harmonics) + if skill.empty: + print("No overlapping real test data for any fitted station.") + else: + print(skill.to_string(index=False)) + n_beat = int(skill["climatology_beats_flat_mean"].sum()) + print(f"\nClimatology beats the fair (train-only) flat-mean baseline on " + f"{n_beat}/{len(skill)} station(s).") + print() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/tests/test_climate_climatology_grid.py b/scripts/tests/test_climate_climatology_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..3891e8700c2bf8046665070386ee60dc7f956fd2 --- /dev/null +++ b/scripts/tests/test_climate_climatology_grid.py @@ -0,0 +1,110 @@ +""" +Grid-cell version of test_climate_climatology.py -- validates +SAFRANLoader.load_grid + build_climate_grid_timeseries against real +ERA5 data, and re-runs the climatology skill check on the de-duplicated +grid cells directly, instead of on station-interpolated series that +provably duplicate signal across stations sharing a cell (confirmed: +your real station-level run showed several stations with bit-identical +values; the same real coordinates collapse to 11 distinct cells out of +27 stations when checked independently here). + +Usage: + python -m scripts.test_climate_climatology_grid --data-root datasets \ + --train-end 2023-12-31 --test-end 2026-12-31 +""" +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +try: + from src.graph.dynamic_features import build_climate_grid_timeseries, map_stations_to_grid_cells + from src.graph.climate_climatology import fit_harmonic_climatology, evaluate_climatology_skill +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_climate_grid_timeseries, map_stations_to_grid_cells + from src.graph.climate_climatology import fit_harmonic_climatology, evaluate_climatology_skill + + +def main() -> None: + parser = argparse.ArgumentParser(description="Test grid-cell climatology against real ERA5 data") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None, + help="Defaults to /station_elevations.csv -- used only to " + "compute the bbox and the station->cell mapping, not for interpolation") + parser.add_argument("--bbox-margin-deg", type=float, default=0.3, + help="Padding added to the real station coordinate extent before querying " + "the grid, so real ERA5 cells just outside all stations aren't excluded") + parser.add_argument("--train-start", type=str, default="2013-01-01") + parser.add_argument("--train-end", type=str, default="2023-12-31") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--n-harmonics", type=int, default=2) + parser.add_argument("--min-months-covered", type=int, default=8) + args = parser.parse_args() + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + print(f"Loaded {len(nodes_df)} real station(s) from {stations_path}") + + m = args.bbox_margin_deg + bbox = (nodes_df["longitude"].min() - m, nodes_df["latitude"].min() - m, + nodes_df["longitude"].max() + m, nodes_df["latitude"].max() + m) + print(f"Grid bbox (real station extent + {m} deg margin): {bbox}") + + safran_path = args.data_root / "safran" + if not safran_path.exists(): + print(f"No {safran_path} found.") + return + + print(f"Loading real ERA5 GRID data from {safran_path} (this may take a while)...") + try: + climate_dict, grid_coords = build_climate_grid_timeseries(safran_path, bbox, (args.train_start, args.test_end)) + except Exception as e: + print(f"FAILED to load/build grid climate timeseries: {e}") + print("Please share the full traceback.") + raise + + print(f"Loaded {len(climate_dict)} real climate variable(s) across {len(grid_coords)} real grid cell(s)") + assignments = map_stations_to_grid_cells(nodes_df, grid_coords) + n_distinct = assignments.nunique() + print(f"{len(nodes_df)} real stations map onto {n_distinct} distinct real grid cell(s) " + f"({len(nodes_df) - n_distinct} station(s) share a cell with at least one other)") + print() + + train_start, train_end, test_end = pd.Timestamp(args.train_start), pd.Timestamp(args.train_end), pd.Timestamp(args.test_end) + + for var, wide in climate_dict.items(): + print("=" * 70) + print(f"Variable: {var} ({wide.shape[1]} real grid cells, vs. {len(nodes_df)} stations before)") + print("=" * 70) + train = wide[(wide.index >= train_start) & (wide.index <= train_end)] + test = wide[(wide.index > train_end) & (wide.index <= test_end)] + + n_real_train = train.notna().sum().sum() + n_real_test = test.notna().sum().sum() + print(f"Real (non-NaN) values -- train: {n_real_train}, test: {n_real_test}") + if n_real_train == 0 or n_real_test == 0: + print("Not enough real data -- skipping.") + print() + continue + + coeffs = fit_harmonic_climatology(train, n_harmonics=args.n_harmonics, + min_months_covered=args.min_months_covered) + print(f"Fitted climatology for {len(coeffs)}/{len(wide.columns)} real grid cell(s)") + if not coeffs: + print() + continue + + train_means = {col: train[col].mean() for col in coeffs.keys()} + skill = evaluate_climatology_skill(test, coeffs, train_means=train_means, n_harmonics=args.n_harmonics) + if not skill.empty: + print(skill.to_string(index=False)) + n_beat = int(skill["climatology_beats_flat_mean"].sum()) + print(f"\nClimatology beats the fair baseline on {n_beat}/{len(skill)} grid cell(s).") + print() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/tests/train_climate_anomaly_model.py b/scripts/tests/train_climate_anomaly_model.py new file mode 100644 index 0000000000000000000000000000000000000000..eb4dfd1a3e74f037328721c10959e3f98eee801b --- /dev/null +++ b/scripts/tests/train_climate_anomaly_model.py @@ -0,0 +1,299 @@ +""" +Trains one ClimateAnomalyModel per climate variable on real ERA5 grid- +cell anomaly data, and evaluates each against the one baseline that +actually matters: predicting a zero anomaly (i.e., trusting climatology +alone). A model that can't beat zero-anomaly at a given horizon isn't +adding real value at that horizon and shouldn't be trusted there. + +Usage: + python -m scripts.train_climate_anomaly_model --data-root datasets \ + --train-end 2023-12-31 --test-end 2026-12-31 +""" +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from torch.utils.data import TensorDataset, DataLoader + +try: + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies, aggregate_anomalies, REGIME_CONFIG + from src.models.climate_anomaly_model import prepare_training_windows, ClimateAnomalyModel, gaussian_nll_loss +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.dynamic_features import build_climate_grid_timeseries + from src.graph.climate_climatology import fit_harmonic_climatology, compute_anomalies, aggregate_anomalies, REGIME_CONFIG + from src.models.climate_anomaly_model import prepare_training_windows, ClimateAnomalyModel, gaussian_nll_loss + + + + +def train_one_variable( + var: str, train_anomalies: pd.DataFrame, val_anomalies: pd.DataFrame, test_anomalies: pd.DataFrame, + horizons: list, lookback_days: int, max_epochs: int, patience: int, batch_size: int, lr: float, + weight_decay: float, device: str, +) -> dict: + print(f"--- Training {var} ---") + + X_train, Y_train, _, _ = prepare_training_windows(train_anomalies, lookback_days, horizons) + X_val, Y_val, _, _ = prepare_training_windows(val_anomalies, lookback_days, horizons) + X_test, Y_test, _, _ = prepare_training_windows(test_anomalies, lookback_days, horizons) + print(f" train examples: {len(X_train)}, val examples: {len(X_val)}, test examples: {len(X_test)}") + + if len(X_train) == 0 or len(X_val) == 0 or len(X_test) == 0: + print(f" not enough data to train {var} -- skipping") + return {"variable": var, "status": "skipped_insufficient_data"} + + X_train_t = torch.tensor(X_train, dtype=torch.float32) + Y_train_t = torch.tensor(Y_train, dtype=torch.float32) + X_val_t = torch.tensor(X_val, dtype=torch.float32).to(device) + Y_val_t = torch.tensor(Y_val, dtype=torch.float32).to(device) + X_test_t = torch.tensor(X_test, dtype=torch.float32).to(device) + Y_test_t = torch.tensor(Y_test, dtype=torch.float32).to(device) + horizons_t = torch.tensor(horizons, dtype=torch.float32).to(device) + + model = ClimateAnomalyModel(init_tau=30.0).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) + loader = DataLoader(TensorDataset(X_train_t, Y_train_t), batch_size=batch_size, shuffle=True) + + best_val_loss = float("inf") + best_state = None + epochs_without_improvement = 0 + + for epoch in range(max_epochs): + model.train() + total_loss, n_batches = 0.0, 0 + for xb, yb in loader: + xb, yb = xb.to(device), yb.to(device) + optimizer.zero_grad() + mean, log_var = model(xb, horizons_t) + loss = gaussian_nll_loss(mean, log_var, yb) + if torch.isnan(loss): + continue # this batch had zero real targets across every example -- skip, don't corrupt gradients + loss.backward() + optimizer.step() + total_loss += loss.item() + n_batches += 1 + + model.eval() + with torch.no_grad(): + val_mean, val_log_var = model(X_val_t, horizons_t) + val_loss = gaussian_nll_loss(val_mean, val_log_var, Y_val_t) + val_loss_value = val_loss.item() if not torch.isnan(val_loss) else float("inf") + + if val_loss_value < best_val_loss: + best_val_loss = val_loss_value + best_state = {k: v.clone() for k, v in model.state_dict().items()} + epochs_without_improvement = 0 + else: + epochs_without_improvement += 1 + + if n_batches > 0 and (epoch % max(1, max_epochs // 20) == 0 or epoch == max_epochs - 1): + print(f" epoch {epoch+1}/{max_epochs}: train NLL = {total_loss / n_batches:.4f}, " + f"val NLL = {val_loss_value:.4f}, tau = {torch.exp(model.log_tau).item():.1f}" + f"{' (best)' if epochs_without_improvement == 0 else ''}") + + if epochs_without_improvement >= patience: + print(f" stopped early at epoch {epoch+1} -- no validation improvement for {patience} epochs") + break + + if best_state is not None: + model.load_state_dict(best_state) + print(f" best validation NLL: {best_val_loss:.4f}") + + model.eval() + with torch.no_grad(): + pred_mean_test, pred_log_var_test = model(X_test_t, horizons_t) + + results = [] + for i, h in enumerate(horizons): + target_h = Y_test_t[:, i] + pred_h = pred_mean_test[:, i] + mask = ~torch.isnan(target_h) + n_valid = int(mask.sum().item()) + if n_valid == 0: + continue + model_rmse = torch.sqrt(((pred_h[mask] - target_h[mask]) ** 2).mean()).item() + zero_anomaly_rmse = torch.sqrt((target_h[mask] ** 2).mean()).item() # predicting anomaly=0, i.e. pure climatology + mean_predicted_variance = torch.exp(pred_log_var_test[:, i][mask]).mean().item() + results.append({ + "horizon_days": h, "n_test_examples": n_valid, + "model_rmse": model_rmse, "climatology_only_rmse": zero_anomaly_rmse, + "model_beats_climatology": model_rmse < zero_anomaly_rmse, + "mean_predicted_std": mean_predicted_variance ** 0.5, + }) + + skill_df = pd.DataFrame(results) + print(skill_df.to_string(index=False)) + print() + + return { + "variable": var, "status": "trained", "tau_learned": torch.exp(model.log_tau).item(), + "best_val_nll": best_val_loss, "skill_by_horizon": results, "model_state": model.state_dict(), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Train the climate anomaly model on real ERA5 data") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--stations-path", type=Path, default=None) + parser.add_argument("--bbox-margin-deg", type=float, default=0.3) + parser.add_argument("--train-start", type=str, default="2013-01-01") + parser.add_argument("--train-end", type=str, default="2021-12-31", + help="End of the actual training period. Default leaves 2022-2023 for " + "validation and 2024+ for the final held-out test, so all three " + "splits are genuinely distinct periods, not overlapping.") + parser.add_argument("--val-end", type=str, default="2023-12-31", + help="End of the validation period, used for early stopping only -- " + "never seen by the optimizer directly.") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--regimes", type=str, nargs="+", default=list(REGIME_CONFIG.keys()), + help="Which resolution regimes to train (default: all of daily/weekly/monthly). " + "See REGIME_CONFIG in climate_climatology.py for each regime's resolution, " + "lookback, and native-unit horizon list.") + parser.add_argument("--n-harmonics", type=int, default=2) + parser.add_argument("--min-months-covered", type=int, default=8) + parser.add_argument("--max-epochs", type=int, default=200, + help="Upper bound only -- early stopping (see --patience) will " + "typically stop well before this for most variables.") + parser.add_argument("--patience", type=int, default=10, + help="Stop training if validation NLL hasn't improved for this many epochs.") + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--weight-decay", type=float, default=1e-4, + help="L2 regularization strength -- pushes the model toward smaller, " + "less confident outputs by default, complementing the decay gate " + "and the NLL loss's own built-in incentive to admit uncertainty.") + parser.add_argument("--variables", type=str, nargs="+", default=None, + help="Restrict to specific variable names (e.g. --variables precip_mm temp_C). " + "Default: train on every real variable found.") + parser.add_argument("--output-dir", type=Path, default=None, + help="Defaults to /climate_anomaly_models") + args = parser.parse_args() + + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using device: {device}") + + stations_path = args.stations_path or (args.data_root / "station_elevations.csv") + nodes_df = pd.read_csv(stations_path) + m = args.bbox_margin_deg + bbox = (nodes_df["longitude"].min() - m, nodes_df["latitude"].min() - m, + nodes_df["longitude"].max() + m, nodes_df["latitude"].max() + m) + + print(f"Loading real ERA5 grid data (bbox={bbox})...") + climate_dict, grid_coords = build_climate_grid_timeseries( + args.data_root / "safran", bbox, (args.train_start, args.test_end) + ) + print(f"Loaded {len(climate_dict)} real variable(s) across {len(grid_coords)} real grid cell(s)") + print() + + train_start, train_end, val_end, test_end = ( + pd.Timestamp(args.train_start), pd.Timestamp(args.train_end), + pd.Timestamp(args.val_end), pd.Timestamp(args.test_end) + ) + + # Confirmed as a real failure mode, not a hypothetical: overriding + # --train-end and --test-end while leaving --val-end at its default + # silently produced an impossible test window (date > val_end AND + # date <= test_end, with val_end already past test_end) -- every + # variable trained "successfully" on an empty test split with zero + # examples and no error at all. Failing loudly here instead of + # letting that happen silently again. + if not (train_start < train_end < val_end < test_end): + raise ValueError( + f"Date ranges must be strictly increasing: train_start < train_end < val_end < test_end.\n" + f"Got: train_start={train_start.date()}, train_end={train_end.date()}, " + f"val_end={val_end.date()}, test_end={test_end.date()}\n" + f"If you're overriding --train-end and/or --test-end, you likely also need to pass " + f"--val-end explicitly -- it doesn't automatically move with the others." + ) + + variables = args.variables or list(climate_dict.keys()) + output_dir = args.output_dir or (args.data_root / "climate_anomaly_models") + output_dir.mkdir(parents=True, exist_ok=True) + + all_results = {} + for var in variables: + if var not in climate_dict: + print(f"'{var}' not found among loaded variables ({list(climate_dict.keys())}), skipping") + continue + + wide = climate_dict[var] + train_wide = wide[(wide.index >= train_start) & (wide.index <= train_end)] + val_wide = wide[(wide.index > train_end) & (wide.index <= val_end)] + test_wide = wide[(wide.index > val_end) & (wide.index <= test_end)] + + # Climatology fit ONLY on the training period, at daily resolution + # regardless of which regime consumes it below -- validation must + # be genuinely held out from every stage, and daily climatology is + # the single source every regime's anomaly is computed relative to + # (see aggregate_anomalies's docstring for why refitting per + # resolution isn't needed). + coeffs = fit_harmonic_climatology(train_wide, n_harmonics=args.n_harmonics, + min_months_covered=args.min_months_covered) + if not coeffs: + print(f"'{var}': no cell had enough real data/seasonal coverage for climatology, skipping") + continue + + train_daily_anomalies = compute_anomalies(train_wide, coeffs, args.n_harmonics) + val_daily_anomalies = compute_anomalies(val_wide, coeffs, args.n_harmonics) + test_daily_anomalies = compute_anomalies(test_wide, coeffs, args.n_harmonics) + + # One model per (variable, regime), not one model spanning all + # horizons -- daily resolution is the structurally wrong target + # past ~2 weeks (see REGIME_CONFIG's docstring); each regime gets + # its own appropriately-aggregated anomaly series, its own native- + # unit horizon list, and its own saved model/skill table. + for regime_name, regime in REGIME_CONFIG.items(): + if regime_name not in args.regimes: + continue + if regime["freq"] is None: + train_anomalies = train_daily_anomalies + val_anomalies = val_daily_anomalies + test_anomalies = test_daily_anomalies + else: + train_anomalies = aggregate_anomalies(train_daily_anomalies, regime["freq"]) + val_anomalies = aggregate_anomalies(val_daily_anomalies, regime["freq"]) + test_anomalies = aggregate_anomalies(test_daily_anomalies, regime["freq"]) + + print(f"[{var} / {regime_name}] resolution={regime['freq'] or 'daily'}, " + f"lookback={regime['lookback']}, native horizons={regime['horizons_native']}") + + result = train_one_variable( + f"{var}_{regime_name}", train_anomalies, val_anomalies, test_anomalies, + regime["horizons_native"], regime["lookback"], + args.max_epochs, args.patience, args.batch_size, args.lr, args.weight_decay, device, + ) + all_results[f"{var}_{regime_name}"] = result + + if result["status"] == "trained": + model_path = output_dir / f"{var}_{regime_name}_anomaly_model.pt" + torch.save(result["model_state"], model_path) + print(f"Saved {var}/{regime_name} model to {model_path}") + + summary = { + var: {"status": r["status"], "tau_learned": r.get("tau_learned"), + "best_val_nll": r.get("best_val_nll"), "skill_by_horizon": r.get("skill_by_horizon")} + for var, r in all_results.items() + } + summary_path = output_dir / "training_summary.json" + summary_path.write_text(json.dumps(summary, indent=2, default=str)) + print(f"\nSaved training summary to {summary_path}") + + print("\n" + "=" * 70) + print("OVERALL: does the anomaly model beat pure climatology, by horizon?") + print("=" * 70) + for var, r in all_results.items(): + if r["status"] != "trained": + continue + beats = [row["horizon_days"] for row in r["skill_by_horizon"] if row["model_beats_climatology"]] + print(f"{var}: beats climatology at horizons {beats}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/training/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc b/scripts/training/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7cc40cb39c92623fccc28822048ed9a7a2e3924f --- /dev/null +++ b/scripts/training/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a54bb608b5e1626b3243cf3c627f452242ffb421b66816bd29b3e9bfe84fcd87 +size 106555 diff --git a/scripts/training/train_spatiotemporal_gnn.py b/scripts/training/train_spatiotemporal_gnn.py new file mode 100644 index 0000000000000000000000000000000000000000..8727cfe8eff47d1852c41cdb0bb858727ae45e86 --- /dev/null +++ b/scripts/training/train_spatiotemporal_gnn.py @@ -0,0 +1,1701 @@ +""" +Trains the main SpatiotemporalGNN on both basins together (shared +backbone), combining: + - real reach graph topology (build_reach_graphs.py output) + - genuine [n_nodes, T] dynamic tensors (dynamic_features.py) + - quantile regression (discharge and water level), not a point estimate + - physics losses, horizon-dependently weighted -- routing weighted + toward short horizons (travel-time lag is hours to days on this + network), water balance toward the same short contiguous block, + confluence/split-rejoin/Manning's flat across every horizon + (structural constraints, not horizon-dependent) + +Usage: + python -m scripts.training.train_spatiotemporal_gnn --data-root datasets +""" +import argparse +import shutil +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +import torch +from torch.utils.data import Dataset, DataLoader + +try: + from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN, masked_mse_graph + from src.graph.pinball_loss import multi_quantile_loss + from src.graph.dynamic_features import ( + build_discharge_timeseries, build_waterlevel_timeseries, build_groundwater_timeseries, + build_precipitation_timeseries, assemble_dynamic_tensor, compute_rolling_precip_sum, + fit_specific_discharge_coefficient, estimate_discharge_from_precip, + build_forecast_tensor, FORECAST_LEAD_TIMES, compute_days_since_last_real, compress_days_since_last_real, + ) + from src.graph.physics_losses import ( + build_confluence_index, build_braid_index, build_routing_index, + confluence_mass_balance_loss, split_rejoin_conservation_loss, routing_consistency_loss, + manning_consistency_loss, water_balance_loss, routing_soft_dtw_loss, + nse_loss, compute_per_node_historical_mean, + ) + from src.graph.subgraph_selection import build_collapsed_subgraph +except ImportError: + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN, masked_mse_graph + from src.graph.pinball_loss import multi_quantile_loss + from src.graph.dynamic_features import ( + build_discharge_timeseries, build_waterlevel_timeseries, build_groundwater_timeseries, + build_precipitation_timeseries, assemble_dynamic_tensor, compute_rolling_precip_sum, + fit_specific_discharge_coefficient, estimate_discharge_from_precip, + build_forecast_tensor, FORECAST_LEAD_TIMES, compute_days_since_last_real, compress_days_since_last_real, + ) + from src.graph.physics_losses import ( + build_confluence_index, build_braid_index, build_routing_index, + confluence_mass_balance_loss, split_rejoin_conservation_loss, routing_consistency_loss, + manning_consistency_loss, water_balance_loss, routing_soft_dtw_loss, + nse_loss, compute_per_node_historical_mean, + ) + from src.graph.subgraph_selection import build_collapsed_subgraph + +BASIN_FILE_NAMES = {0: "eure", 1: "risle"} + +# Must match SpatiotemporalGNN's default quantile list exactly -- index 0 +# is treated as the median everywhere physics losses extract a single +# expected-value slice (see compute_physics_loss). +QUANTILES = [0.5, 0.9, 0.95, 0.99] + +# Contiguous daily block first (routing_consistency_loss needs uniformly +# daily-spaced steps -- its lag is in units of "one array step", which +# only corresponds to real calendar days if the steps genuinely are +# daily), then real forecast-coverage-relevant horizons out to 21 days. +# Deliberately narrowed from an earlier [..., 30, 60, 90, 180] range -- +# every one of those longer horizons showed the same real pattern +# throughout this project: zero real events in the real test window at +# 60-180 days (too short a real test period to contain any), and even +# where real events existed (30d), precision sat far below the real +# base rate -- real evidence of hedging, not genuine prediction, not a +# horizon worth spending real model capacity on. 21 is the real, +# deliberate ceiling matching this project's planned future real +# Météo-France forecast horizon, not an arbitrary round number. +HORIZONS = [1, 2, 3, 4, 5, 7, 10, 14, 21] +ROUTING_HORIZON_COUNT = 7 # first 7 days -- extended from 5 to match the real max ECMWF +# forecast lead time (7, now in HORIZONS) -- routing_consistency_loss and Soft-DTW compare +# the model's own predictions against each other, so extending them carries no real new +# risk. water_balance_loss is the one term this extension is more borderline for (it +# checks predicted discharge against HISTORICAL, already-observed precipitation, which +# genuinely stops explaining discharge the further out the prediction horizon goes) -- 7 +# is still within a real, defensible range; this constant should not be pushed further +# without separately reconsidering water_balance_loss's own real physical justification. + + +def get_dynamic_channel_index(var_names: List[str], target_var: str, channels_per_var: int = 2) -> Tuple[int, int]: + """ + (value_index, missing_index) for a given dynamic variable within + x_gauge_dynamic_seq's channel axis -- derived explicitly from + var_names (the same dict key order prepare_graph_training_windows + itself iterates over to build channels, in insertion order), not + hardcoded. A hardcoded index would silently break the moment + dynamic_tensors' key order changed for any reason (e.g. adding a + new variable before precipitation), which is exactly the kind of + subtle bug this project has caught and fixed several times before + in this same file. + + channels_per_var must match whatever prepare_graph_training_windows + was actually called with -- 2 (value, missing) if dt_tensors was + None, 3 (value, missing, days_since_real) if it was provided. Using + the wrong count here wouldn't raise an error; it would silently + point at the wrong channel entirely, so this is a real, easy + mistake to make quietly -- pass it explicitly at every call site + rather than relying on the default matching by coincidence. + """ + if target_var not in var_names: + raise ValueError(f"'{target_var}' not found in {var_names}") + position = var_names.index(target_var) + return channels_per_var * position, channels_per_var * position + 1 + + +def extract_weekly_peak_targets_at_anchor_dates( + target_tensor: np.ndarray, dates: List[pd.Timestamp], anchor_dates: List[pd.Timestamp], + week_horizons: List[int], window_size: int = 7, +) -> np.ndarray: + """ + A genuinely different kind of target from extract_targets_at_anchor_ + dates above -- not a point-in-time value, but the real PEAK within a + non-overlapping window of days. Built specifically because a real + flood is a multi-day event (confirmed against real data via + identify_flood_events.py: real event durations run 3-10 days), and + averaging or point-sampling across a week would dilute or could + entirely miss it -- the same reasoning that ruled out monthly + aggregation for flood detection specifically, applied here at a + scale (weekly) where the event and the window are actually + comparable in length, not wildly mismatched. + + Non-overlapping, not rolling: week_horizons=[7,14,21,28] means + "the real peak in days 1-7", "the real peak in days 8-14", and so + on -- each real day contributes to exactly one weekly target, not + several overlapping ones, so a single real flood doesn't get + counted (or missed) multiple times across adjacent windows. + + Mirrors extract_targets_at_anchor_dates' own date-indexing exactly, + just replacing a single t+h-1 point lookup with a real max() over + the h-window_size..h-1 real day range. + """ + date_to_idx = {d: i for i, d in enumerate(dates)} + rows = [] + for anchor in anchor_dates: + t = date_to_idx[anchor] + weekly_peaks = [] + for h in week_horizons: + window = target_tensor[:, t + h - window_size: t + h] # [n_nodes, window_size] + # nanmax over a window that's entirely NaN correctly stays + # NaN (with a warning numpy emits and we accept, matching + # the same "missing stays missing" principle used + # throughout this project) rather than silently becoming 0 + # or some other fabricated fallback. + with np.errstate(invalid="ignore"): + peak = np.nanmax(window, axis=1) if window.shape[1] > 0 else np.full(window.shape[0], np.nan) + all_nan_mask = np.all(np.isnan(window), axis=1) + peak = np.where(all_nan_mask, np.nan, peak) + weekly_peaks.append(peak) + rows.append(np.stack(weekly_peaks, axis=-1)) + return np.stack(rows) + + +def extract_targets_at_anchor_dates( + target_tensor: np.ndarray, dates: List[pd.Timestamp], anchor_dates: List[pd.Timestamp], horizons: List[int], +) -> np.ndarray: + """ + Extracts target values at a FIXED, already-determined set of anchor + dates -- deliberately not re-running the full window-selection + filter (which only keeps anchor dates with at least one real value + ANYWHERE in the graph) a second time for a different target + variable. Water level and discharge have genuinely different real + coverage (13 stations vs 6-8 in-window), so independently filtering + each would very plausibly produce two different-length, misaligned + sets of examples -- this guarantees perfect alignment with whatever + anchor_dates discharge's own (already-tested) window preparation + already settled on, by construction, not by coincidence. + + Mirrors prepare_graph_training_windows' own t+h-1 indexing exactly + (see spatiotemporal_gnn.py) -- same real target-extraction logic, + just applied to a caller-supplied date list instead of discovering + its own. + """ + date_to_idx = {d: i for i, d in enumerate(dates)} + rows = [] + for anchor in anchor_dates: + t = date_to_idx[anchor] + targets = np.stack([target_tensor[:, t + h - 1] for h in horizons], axis=-1) + rows.append(targets) + return np.stack(rows) + + +def load_basin_graph(data_root: Path, file_key: str) -> Tuple[pd.DataFrame, pd.DataFrame]: + graph_dir = data_root / "reach_graph" + nodes_path = graph_dir / f"{file_key}_nodes_enriched.csv" + edges_path = graph_dir / f"{file_key}_edges.csv" + if not nodes_path.exists() or not edges_path.exists(): + raise FileNotFoundError(f"Missing {nodes_path} or {edges_path} -- run build_reach_graphs.py + " + f"enrich_reach_graph.py first.") + nodes_df, edges_df = pd.read_csv(nodes_path), pd.read_csv(edges_path) + + widths_path = data_root / "edge_widths.csv" + if widths_path.exists() and "cleabs" in edges_df.columns: + widths_df = pd.read_csv(widths_path)[["cleabs", "width_m"]] + edges_df = edges_df.merge(widths_df, on="cleabs", how="left") + n_real = edges_df["width_m"].notna().sum() + print(f" {file_key}: joined real channel width for {n_real}/{len(edges_df)} edge(s) " + f"from {widths_path}") + else: + print(f" {file_key}: no {widths_path} found (or edges table has no 'cleabs' column) -- " + f"run compute_edge_width.py first if you want real channel width in edge_attr. " + f"Proceeding without it (width_m absent -> filled+flagged missing downstream).") + + return nodes_df, edges_df + + +def combine_basins( + basin_data: List[Tuple[pd.DataFrame, pd.DataFrame, int]], + shared_static_cols: List[str], gauge_static_cols: List[str], +) -> Dict[str, object]: + """ + Combines multiple basins' real node/edge tables into one training + universe -- same shape PyG's Batch.from_data_list produces (node + features concatenated along axis 0, edge indices offset per basin so + they still point at the right rows in the combined table), built in + plain numpy here specifically so it's testable without torch/ + torch_geometric installed. + + Splits static features into shared_static (genuinely present at + every node -- elevation, IDPR, cumulative catchment area) and + gauge_static (landcover/geology/NDVI/cavites -- currently real only + for gauge nodes, 0-filled elsewhere), per the gauge-vs-non-gauge + input design (see spatiotemporal_gnn.py's TypeAwareInputEncoder). + gauge_static_cols missing from a given basin's real enriched table + (e.g. if that basin's enrichment step hasn't added geology yet) are + filled with 0 rather than raising, since a genuinely missing + dataset should degrade gracefully to "no gauge-static signal + available yet," not crash the whole pipeline. + + Args: + basin_data: list of (nodes_df, edges_df, basin_id) tuples. + shared_static_cols, gauge_static_cols: which real columns from + nodes_df go in each group. + + Returns: + dict with combined station_codes, x_shared_static, x_gauge_static, + is_gauged, basin_id_array, edge_index [2, n_edges], + edge_attr [n_edges, 5] (distance_km, elevation_drop_m, + verified_continuous, width_m, width_missing_flag), and + per-basin node offsets (needed to + build dynamic tensors with matching node ordering afterward). + """ + all_codes, all_shared_static, all_gauge_static, all_basin_ids, all_is_gauged = [], [], [], [], [] + all_edge_sources, all_edge_targets, all_edge_attrs = [], [], [] + offsets = {} + running_offset = 0 + + for nodes_df, edges_df, basin_id in basin_data: + offsets[basin_id] = running_offset + code_to_local_idx = {code: i for i, code in enumerate(nodes_df["station_code"])} + + all_codes.extend(nodes_df["station_code"].tolist()) + all_shared_static.append(nodes_df[shared_static_cols].values.astype(np.float32)) + + gauge_static_df = nodes_df.reindex(columns=gauge_static_cols, fill_value=0.0) + all_gauge_static.append(gauge_static_df.values.astype(np.float32)) + + is_gauged_col = nodes_df["is_gauged"] if "is_gauged" in nodes_df.columns else pd.Series([False] * len(nodes_df)) + all_is_gauged.extend(is_gauged_col.astype(bool).tolist()) + + all_basin_ids.extend([basin_id] * len(nodes_df)) + + for _, e in edges_df.iterrows(): + if e["source"] not in code_to_local_idx or e["target"] not in code_to_local_idx: + continue + all_edge_sources.append(code_to_local_idx[e["source"]] + running_offset) + all_edge_targets.append(code_to_local_idx[e["target"]] + running_offset) + + # width_m can be genuinely missing (no real BD TOPO surface polygon + # matched that tronçon -- see compute_edge_width.py) or absent + # entirely if that script hasn't been run for this basin yet. + # NaN flowing directly into the message-passing edge encoder + # would poison the whole computation, so it's filled + flagged + # explicitly here -- the same missing-data discipline already + # used throughout this project for dynamic node features + # (prepare_graph_training_windows's value+missingness channels), + # not a new pattern invented for this one field. + width_raw = e["width_m"] if "width_m" in edges_df.columns else np.nan + width_missing = pd.isna(width_raw) + width_filled = 0.0 if width_missing else float(width_raw) + + all_edge_attrs.append([e["distance_km"], e["elevation_drop_m"], + float(e.get("verified_continuous", True)), + width_filled, float(width_missing)]) + + running_offset += len(nodes_df) + + return { + "station_codes": all_codes, + "x_shared_static": np.concatenate(all_shared_static, axis=0), + "x_gauge_static": np.concatenate(all_gauge_static, axis=0), + "is_gauged": np.array(all_is_gauged, dtype=bool), + "basin_id": np.array(all_basin_ids, dtype=np.int64), + "edge_index": np.array([all_edge_sources, all_edge_targets], dtype=np.int64), + "edge_attr": np.array(all_edge_attrs, dtype=np.float32), + "node_offsets": offsets, + "n_nodes": running_offset, + } + + +def extract_catchment_area_km2(basin_data: List[Tuple[pd.DataFrame, pd.DataFrame, int]], combined: Dict) -> np.ndarray: + """ + Real per-node cumulative catchment area, in the same combined node + order as everything else in `combined` -- built separately from + x_gauge_static deliberately, not folded into that pathway's + fill-missing-with-0 convention. 0 km2 is not a safe placeholder + here the way it is for other gauge-static features: water_balance_ + loss multiplies precipitation by this value, so a fabricated 0 + would silently zero out that node's entire water-balance + contribution rather than correctly excluding it. Genuine NaN is + preserved for any node missing real catchment data, which + water_balance_loss's own existing NaN-exclusion logic already + handles correctly. + """ + n_nodes = combined["n_nodes"] + areas = np.full(n_nodes, np.nan, dtype=np.float32) + for nodes_df, _, basin_id in basin_data: + offset = combined["node_offsets"][basin_id] + primary = nodes_df["cumulative_catchment_area_km2"].values if "cumulative_catchment_area_km2" in nodes_df.columns \ + else np.full(len(nodes_df), np.nan) + # Real fallback, not a fabricated one -- confirmed directly + # against the real enriched CSVs (cumulative_catchment_area_km2 + # is entirely absent -- compute_cumulative_catchment.py's + # output was never merged in), while catchment_area_km2 (the + # older, Hub'Eau-sourced column, real but only populated for + # actual gauge stations, not confluences/braid points) IS + # present. Falls back PER NODE, not per column: a node could + # have the cumulative value but not the Hub'Eau one, or vice + # versa, so this isn't simply "prefer column A over column B" + # wholesale. + fallback = nodes_df["catchment_area_km2"].values if "catchment_area_km2" in nodes_df.columns \ + else np.full(len(nodes_df), np.nan) + combined_area = np.where(~np.isnan(primary), primary, fallback) + areas[offset:offset + len(nodes_df)] = combined_area + # Both columns genuinely absent (enrichment not run at all for + # this basin) -- areas stays NaN for these nodes, degrading + # gracefully rather than crashing, same principle as + # precipitation's optional_vars handling. + return areas + + +def build_combined_dynamic_tensors( + combined: Dict, basin_data: List[Tuple[pd.DataFrame, pd.DataFrame, int]], + data_root: Path, date_range: Tuple[str, str], +) -> Tuple[Dict[str, np.ndarray], np.ndarray, np.ndarray, List[pd.Timestamp]]: + """ + Builds real discharge (target), water level (target -- see below), + and groundwater (input) tensors per basin, then places each basin's + real values into the correct rows of one combined [n_nodes_total, T] + array using node_offsets -- so the combined tensors' row ordering + matches combined['x_shared_static'] exactly, node for node. + + Water level is a SECOND real target, not previously wired in at all + -- the model's water-level output channel had zero direct + supervision before this (only one loose consistency constraint, + manning_consistency_loss), which is exactly why the earlier learned + bank-height value (0.30m) looked physically implausible: nothing + had ever told that output channel what a real water level actually + looks like. Confirmed real coverage is DIFFERENT from discharge's + (13 stations have real water-level data vs discharge's 6-8 + in-window stations) -- not the same station set, so this doesn't + just duplicate the existing discharge supervision under a new name. + """ + n_nodes = combined["n_nodes"] + dates = None + discharge_combined = None + waterlevel_combined = None + gw_level_combined = None + precip_combined = None + + for nodes_df, _, basin_id in basin_data: + offset = combined["node_offsets"][basin_id] + + discharge_wide = build_discharge_timeseries(nodes_df, data_root / "hydrometric", date_range) + discharge_tensor, basin_dates = assemble_dynamic_tensor(nodes_df, discharge_wide) + if dates is None: + dates = basin_dates + discharge_combined = np.full((n_nodes, len(dates)), np.nan, dtype=np.float32) + waterlevel_combined = np.full((n_nodes, len(dates)), np.nan, dtype=np.float32) + gw_level_combined = np.full((n_nodes, len(dates)), np.nan, dtype=np.float32) + precip_combined = np.full((n_nodes, len(dates)), np.nan, dtype=np.float32) + discharge_combined[offset:offset + len(nodes_df), :] = discharge_tensor + + waterlevel_wide = build_waterlevel_timeseries(nodes_df, data_root / "hydrometric", date_range) + waterlevel_tensor, _ = assemble_dynamic_tensor(nodes_df, waterlevel_wide) + waterlevel_combined[offset:offset + len(nodes_df), :] = waterlevel_tensor + + level_wide, _ = build_groundwater_timeseries(nodes_df, data_root / "ades", date_range) + level_tensor, _ = assemble_dynamic_tensor(nodes_df, level_wide) + gw_level_combined[offset:offset + len(nodes_df), :] = level_tensor + + # Real, directly-measured precipitation (Météo-France gauges), + # not ERA5 reanalysis -- input only, no target here, so a + # missing precipitation_daily.csv degrades gracefully to an + # all-NaN channel (handled by the same value+missingness fill + # pattern already used for every other dynamic input) rather + # than blocking training entirely. + precip_csv_path = data_root / "meteofrance_precipitation" / "daily_precipitation.csv" + if precip_csv_path.exists(): + precip_wide = build_precipitation_timeseries(nodes_df, precip_csv_path, date_range) + precip_tensor, _ = assemble_dynamic_tensor(nodes_df, precip_wide) + precip_combined[offset:offset + len(nodes_df), :] = precip_tensor + + dynamic_tensors = {"discharge": discharge_combined, "groundwater_level": gw_level_combined} + # Only added as a real key if it actually has real data -- confirmed + # as a real bug, not just a style nit: unconditionally adding + # "precipitation" with an all-NaN array meant every downstream + # `if "precipitation" in dynamic_tensors` check always evaluated + # True regardless of whether real data was ever found, making the + # "no real precipitation data" messaging misleading about what + # actually happened. + if not np.isnan(precip_combined).all(): + dynamic_tensors["precipitation"] = precip_combined + return dynamic_tensors, discharge_combined, waterlevel_combined, dates + + +def standardize_static_features( + x_static: np.ndarray, col_names: List[str], +) -> Tuple[np.ndarray, Dict[str, Tuple[float, float]], List[str]]: + """ + Z-scores each static feature column using its own real (non-NaN) + mean/std across all real nodes, then safe-fills any remaining NaN + to 0 (the same "0 = at the mean / no information" convention used + for dynamic features) -- paired with an explicit missingness flag + for any column that has real NaN, so the model can distinguish a + genuinely average value from a genuinely unknown one. + + No train/val/test split is applied here, unlike dynamic feature + standardization: static node characteristics have no time dimension + to leak across -- every node participates in both training and + validation examples, just at different anchor dates, so fitting + statistics across all real values is the correct scope, not a + leakage risk the way it would be for a temporal split. + + A missingness flag column is only added for a feature that actually + has real NaN somewhere, not unconditionally for every feature -- + avoids adding a dead, always-zero column for a feature (elevation, + IDPR) that's always fully populated. + + Args: + x_static: [n_nodes, n_features], real NaN allowed. + col_names: names for each column, in order. + + Returns: + x_out: [n_nodes, n_features + n_flagged], standardized values + followed by missingness flags for whichever columns needed one. + stats: {col_name: (mean, std)}. + flagged_names: names of the columns that received a missingness + flag, in the order their flag columns were appended. + """ + n_nodes, n_features = x_static.shape + out_cols = [] + stats = {} + flag_cols = [] + flagged_names = [] + + for i, name in enumerate(col_names): + col = x_static[:, i] + is_nan = np.isnan(col) + real_vals = col[~is_nan] + if len(real_vals) == 0: + # Every value is NaN -- nothing real to standardize against. + # Output stays 0 everywhere, flagged as entirely missing + # rather than silently treated as a real, populated feature. + mean, std = 0.0, 1.0 + standardized = np.zeros(n_nodes, dtype=np.float32) + else: + mean = float(real_vals.mean()) + std = float(real_vals.std()) + if std < 1e-8: + std = 1.0 # a real, constant-valued feature -- avoid dividing by ~0 + standardized = np.where(is_nan, 0.0, (col - mean) / std).astype(np.float32) + stats[name] = (mean, std) + out_cols.append(standardized) + if is_nan.any(): + flag_cols.append(is_nan.astype(np.float32)) + flagged_names.append(name) + + if not out_cols and not flag_cols: + # No real columns given at all (e.g. gauge_static_cols + # deliberately left empty) -- np.stack has no defined behavior + # for zero arrays, so this degenerate case is handled explicitly + # rather than left to fail inside the library call. + x_out = np.zeros((n_nodes, 0), dtype=np.float32) + else: + x_out = np.stack(out_cols + flag_cols, axis=1) + return x_out, stats, flagged_names + + +def standardize_dynamic_tensors( + dynamic_tensors: Dict[str, np.ndarray], target_tensor: np.ndarray, waterlevel_target_tensor: np.ndarray, + dates: List[pd.Timestamp], train_end: str, optional_vars: Optional[List[str]] = None, +) -> Tuple[Dict[str, np.ndarray], np.ndarray, np.ndarray, Dict[str, Tuple[float, float]]]: + """ + Z-scores every dynamic variable (and the discharge target, which is + literally the same underlying real values as dynamic_tensors + ["discharge"] -- standardizing dynamic_tensors["discharge"] in place + correctly standardizes target_tensor too, since it's the same array), + PLUS the water-level target -- fitted independently, on its own real + training-period statistics, not reusing discharge's stats. Water + level is mm-scale and physically unrelated to discharge's L/s scale, + so sharing one mean/std between them would be wrong on its own + terms, not just a minor inconsistency. + + Fit on the TRAINING period only (dates <= train_end), applied to the + whole tensor -- avoids leaking val/test statistics into training, + same discipline already used for climatology fitting throughout this + project. + + optional_vars: variables allowed to be entirely missing (e.g. + "precipitation" -- real, but only wired in if + fetch_meteofrance_precipitation.py has actually been run) without + raising. Real gap this closes: build_combined_dynamic_tensors + already degrades gracefully to an all-NaN channel when the real + precipitation CSV doesn't exist yet, but without this, that all-NaN + channel would still hit the hard error below and crash the whole + pipeline -- turning an optional, not-yet-fetched input into a fatal + error. Non-optional variables (discharge, groundwater) keep the + hard error: zero real data for THOSE is a genuine bug worth + surfacing loudly, not something to silently paper over. + + Returns: + standardized dynamic_tensors, standardized discharge target, + standardized water-level target, and the {variable: (mean, std)} + used -- needed later to convert predictions/losses back to real + physical units for reporting. + """ + optional_vars = optional_vars or [] + dates_arr = pd.DatetimeIndex(dates) + train_end_ts = pd.Timestamp(train_end) + train_col_mask = dates_arr <= train_end_ts + + stats = {} + standardized = {} + for var, tensor in dynamic_tensors.items(): + train_values = tensor[:, train_col_mask] + real_train_values = train_values[~np.isnan(train_values)] + if len(real_train_values) == 0: + if var in optional_vars: + print(f" '{var}': zero real training-period values -- optional variable, " + f"skipping standardization (stays all-NaN, handled by the usual " + f"value+missingness fill downstream, not a fatal error).") + stats[var] = (0.0, 1.0) + standardized[var] = tensor + continue + raise ValueError(f"'{var}' has zero real training-period values -- cannot standardize. " + f"Check --train-end against this variable's real date coverage.") + mean, std = float(real_train_values.mean()), float(real_train_values.std()) + if std == 0: + std = 1.0 # degenerate case guard -- avoid dividing by zero for a constant variable + stats[var] = (mean, std) + standardized[var] = (tensor - mean) / std + + target_mean, target_std = stats.get("discharge", (0.0, 1.0)) + target_standardized = (target_tensor - target_mean) / target_std + + wl_train_values = waterlevel_target_tensor[:, train_col_mask] + wl_real_train_values = wl_train_values[~np.isnan(wl_train_values)] + if len(wl_real_train_values) == 0: + raise ValueError("waterlevel target has zero real training-period values -- cannot standardize. " + "Check --train-end against real water-level date coverage.") + wl_mean, wl_std = float(wl_real_train_values.mean()), float(wl_real_train_values.std()) + if wl_std == 0: + wl_std = 1.0 + stats["waterlevel"] = (wl_mean, wl_std) + waterlevel_target_standardized = (waterlevel_target_tensor - wl_mean) / wl_std + + return standardized, target_standardized, waterlevel_target_standardized, stats + + +def compute_physics_loss( + Q_pred_quantiles: "object", H_pred_quantiles: "object", horizons: List[int], + confluence_index, braid_index, routing_index, + edge_index: "object", edge_attr: "object", manning_n_logit: "object", log_bank_height_m: "object", + weights: Dict[str, float], + recent_precip_sum_mm: "object" = None, catchment_area_km2: "object" = None, + discharge_mean: float = 0.0, discharge_std: float = 1.0, + soft_dtw_gamma: float = 1.0, + debug: bool = False, +) -> "object": + """ + Horizon-dependently weighted combination -- NOT physics_informed_ + loss's single fixed-weight combination directly, since that assumes + one weights dict applies uniformly to every timestep. Confluence and + split/rejoin are structural constraints applied identically at every + horizon; routing only among the contiguous daily block (see + ROUTING_HORIZON_COUNT); Manning's equation applied identically at + every horizon too (a reach's geometry/roughness relationship holds + regardless of lead time, unlike routing's lag which specifically + needs contiguous daily steps). + + Water balance is applied ONLY at the contiguous short-horizon block + (same block routing uses), not every horizon -- a real, deliberate + scope limit, not an oversight. recent_precip_sum_mm is REAL, + HISTORICAL precipitation (summed over the lookback window, already + observed) -- it can only be checked against NEAR-term discharge + predictions, where recent rainfall genuinely still drives discharge + through real catchment storage/lag. It cannot be checked against + longer-horizon predictions (10-21 days out) without real FUTURE + precipitation -- that's what the decoder-side forecast_precip + pathway is for instead (real ECMWF forecast data, wired in + separately, with its own real coverage limits -- see + FORECAST_LEAD_TIMES). Using historical rainfall to check a longer- + horizon prediction would be checking future discharge against + unrelated past rainfall, a real physical mismatch, not just a + weaker constraint. + + evap_mm is passed as 0.0 -- real evapotranspiration data exists + from the separate climate work but lives on ERA5 grid cells, not + this graph's nodes, and wiring that mapping is real, separate work + not done here. This is an explicit, documented simplification, not + a silently dropped term: omitting ET folds it into the residual + water_balance_loss checks against, weakening the constraint but not + invalidating it (ET is real but typically smaller than P and Q at + the short horizons this is actually applied to). + + Every physics term here checks a constraint against the MEDIAN + quantile specifically, not every quantile the model predicts. Mass + conservation and Manning's equation are statements about the + expected physical state -- a confluence's expected downstream flow + equals the sum of its expected upstream flows. That's not + guaranteed to hold for other quantiles: the 95th percentile of a + sum is not generally the sum of the 95th percentiles unless the + branches are perfectly correlated, so enforcing this constraint on + every quantile would be imposing a false physical requirement, not + a real one. + + Args: + Q_pred_quantiles, H_pred_quantiles: [n_nodes, n_horizons, + n_quantiles] -- the full quantile predictions for + discharge/water level (the two target slices of the + model's full [n_nodes, n_horizons, n_targets, n_quantiles] + output). Quantile index 0 is assumed to be the median, per + SpatiotemporalGNN's default quantile ordering. + recent_precip_sum_mm: [n_nodes], optional -- real precipitation + summed over the lookback window. None skips water balance + entirely (e.g. precipitation data not wired in for this run). + catchment_area_km2: [n_nodes], optional real per-node catchment + area -- required alongside recent_precip_sum_mm for water + balance; if either is None, that term is skipped, not + silently zeroed. + debug: if True, prints each term's finiteness/value separately + -- added specifically because reasoning alone couldn't + isolate a real, persistent val-loss=inf failure across + several hypothesis rounds; direct per-term evidence is + needed instead of continuing to guess. + """ + Q_median = Q_pred_quantiles[:, :, 0] + H_median = H_pred_quantiles[:, :, 0] + + total = Q_median.sum() * 0.0 + n_horizons = len(horizons) + + conf_losses, split_losses, manning_losses = [], [], [] + for h_idx in range(n_horizons): + Q_h = Q_median[:, h_idx] + H_h = H_median[:, h_idx] + conf_losses.append(confluence_mass_balance_loss(Q_h, confluence_index)) + split_losses.append(split_rejoin_conservation_loss(Q_h, braid_index)) + manning_losses.append(manning_consistency_loss(Q_h, H_h, edge_index, edge_attr, manning_n_logit, log_bank_height_m)) + conf_term = weights.get("confluence", 1.0) * sum(conf_losses) / n_horizons + split_term = weights.get("split_rejoin", 1.0) * sum(split_losses) / n_horizons + manning_term = weights.get("manning", 1.0) * sum(manning_losses) / n_horizons + if debug: + print(f" [debug] confluence term: {conf_term.item():.6g}, finite={torch.isfinite(conf_term).item()}") + print(f" [debug] split_rejoin term: {split_term.item():.6g}, finite={torch.isfinite(split_term).item()}") + print(f" [debug] manning term: {manning_term.item():.6g}, finite={torch.isfinite(manning_term).item()}") + # Isolate WHICH term's GRADIENT (not just its forward value) is + # the real source -- static code review couldn't pin this down + # after multiple attempts, so measuring it directly per term. + for term_name, term_val in [("confluence", conf_term), ("split_rejoin", split_term), ("manning", manning_term)]: + g_Q, g_H = torch.autograd.grad(term_val, [Q_median, H_median], retain_graph=True, allow_unused=True) + g_Q_ok = g_Q is None or torch.isfinite(g_Q).all().item() + g_H_ok = g_H is None or torch.isfinite(g_H).all().item() + print(f" [debug] GRADIENT of {term_name} term: wrt Q_median finite={g_Q_ok}, wrt H_median finite={g_H_ok}") + total = total + conf_term + split_term + manning_term + + if routing_index and ROUTING_HORIZON_COUNT <= n_horizons: + Q_daily_block = Q_median[:, :ROUTING_HORIZON_COUNT] + routing_loss = routing_consistency_loss(Q_daily_block, routing_index) + routing_term = weights.get("routing", 1.0) * routing_loss + if debug: + print(f" [debug] routing term: {routing_term.item():.6g}, finite={torch.isfinite(routing_term).item()}") + g_Q, = torch.autograd.grad(routing_term, [Q_median], retain_graph=True, allow_unused=True) + g_Q_ok = g_Q is None or torch.isfinite(g_Q).all().item() + print(f" [debug] GRADIENT of routing term: wrt Q_median finite={g_Q_ok}") + total = total + routing_term + + # Complements the fixed-lag routing term above with soft-DTW's + # elastic time-alignment -- tolerant of the calculated lag being + # slightly off, rather than assuming it's exact. + soft_dtw_loss = routing_soft_dtw_loss(Q_daily_block, routing_index, gamma=soft_dtw_gamma) + soft_dtw_term = weights.get("routing_soft_dtw", 1.0) * soft_dtw_loss + if debug: + print(f" [debug] routing_soft_dtw term: {soft_dtw_term.item():.6g}, " + f"finite={torch.isfinite(soft_dtw_term).item()}") + total = total + soft_dtw_term + + if recent_precip_sum_mm is not None and catchment_area_km2 is not None and ROUTING_HORIZON_COUNT <= n_horizons: + # Discharge averaged over the same short block water balance is + # scoped to -- a period-average rate over ROUTING_HORIZON_COUNT + # days, matching water_balance_loss's own period_days convention. + Q_short_block_avg_standardized = Q_median[:, :ROUTING_HORIZON_COUNT].mean(dim=1) + # Confirmed as a real bug against real training, not + # hypothetical: water_balance_loss expects real physical units + # (L/s) throughout, but the model's own output is in + # STANDARDIZED units (it was trained on z-scored targets). This + # is a simple affine conversion (not a sum, unlike precip's own + # fix above), so no per-day subtlety applies here. + Q_short_block_avg = Q_short_block_avg_standardized * discharge_std + discharge_mean + evap_placeholder = Q_short_block_avg * 0.0 # explicit 0.0, see docstring -- not a silent omission + if debug: + print(f" [debug] Q_short_block_avg (real L/s): min={Q_short_block_avg.min().item():.4g}, " + f"max={Q_short_block_avg.max().item():.4g}, finite_all={torch.isfinite(Q_short_block_avg).all().item()}") + real_precip_mask = ~torch.isnan(recent_precip_sum_mm) + if real_precip_mask.any(): + print(f" [debug] recent_precip_sum_mm (real mm): min={recent_precip_sum_mm[real_precip_mask].min().item():.4g}, " + f"max={recent_precip_sum_mm[real_precip_mask].max().item():.4g}, n_real={int(real_precip_mask.sum())}") + else: + print(f" [debug] recent_precip_sum_mm: zero real (non-NaN) values this batch") + real_area_mask = ~torch.isnan(catchment_area_km2) + print(f" [debug] catchment_area_km2: n_real={int(real_area_mask.sum())}, " + f"min={catchment_area_km2[real_area_mask].min().item() if real_area_mask.any() else float('nan'):.4g}, " + f"max={catchment_area_km2[real_area_mask].max().item() if real_area_mask.any() else float('nan'):.4g}") + wb_loss = water_balance_loss( + Q_short_block_avg, recent_precip_sum_mm, evap_placeholder, catchment_area_km2, + period_days=float(ROUTING_HORIZON_COUNT), + ) + if debug: + print(f" [debug] water_balance term (pre-weight): {wb_loss.item():.6g}, finite={torch.isfinite(wb_loss).item()}") + g_Q, = torch.autograd.grad(wb_loss, [Q_median], retain_graph=True, allow_unused=True) + g_Q_ok = g_Q is None or torch.isfinite(g_Q).all().item() + print(f" [debug] GRADIENT of water_balance term: wrt Q_median finite={g_Q_ok}") + if not torch.isfinite(wb_loss): + pass # skip contribution -- don't let a non-finite water-balance term poison the total + else: + total = total + weights.get("water_balance", 1.0) * wb_loss + + return total + + +class GraphWindowDataset(Dataset): + """ + Thin wrapper over the (X, Y, Y_mask, Y_H, Y_H_mask, forecast_precip, + forecast_missing) arrays -- x_shared_static/x_gauge_static/edge_index/ + edge_attr/basin_id are the SAME real graph for every example (only + the dynamic history varies per anchor date), so they're not part of + this dataset; the training loop holds them once and reuses them. + + Y_H/Y_H_mask are the real water-level targets, extracted at the + exact same anchor dates as Y (see extract_targets_at_anchor_dates) + -- perfectly aligned with Y by construction, not by coincidence. + + forecast_precip/forecast_missing are real forecasted precipitation + per (example, node, horizon) -- see build_forecast_tensor. Real + coverage is genuinely partial (only horizons [1,2,3,5,7], only from + 2024-03-01 onward), so forecast_missing is mostly True across this + project's full 2013+ training period -- that's the real, honest + shape of this data, not a bug. + + Daily only -- weekly-peak targets were tried and reverted: joint + training on daily and weekly objectives simultaneously (running the + model twice per example, on two competing losses) coincided with a + real, measured drop in daily discharge skill (naive-baseline beat + dropped from ~35% to 6.6% in the one real run this was tested on). + Genuinely too many other things changed in that same run (a shifted + train/val split, new Δt channels) to say for certain the weekly + objective specifically caused it, but reverting removes it as a + variable rather than keep training on an unresolved regression. + Longer-range context, if needed later, has a real simpler path: + more real precipitation history as an input (Météo-France's archive + covers a 21-day lookback without any of this dual-objective + complexity), not a second competing prediction target. + """ + + def __init__( + self, X: np.ndarray, Y: np.ndarray, Y_mask: np.ndarray, Y_H: np.ndarray, Y_H_mask: np.ndarray, + forecast_precip: np.ndarray, forecast_missing: np.ndarray, + ): + self.X = torch.tensor(X, dtype=torch.float32) + self.Y = torch.tensor(Y, dtype=torch.float32) + self.Y_mask = torch.tensor(Y_mask, dtype=torch.bool) + self.Y_H = torch.tensor(Y_H, dtype=torch.float32) + self.Y_H_mask = torch.tensor(Y_H_mask, dtype=torch.bool) + self.forecast_precip = torch.tensor(forecast_precip, dtype=torch.float32) + self.forecast_missing = torch.tensor(forecast_missing, dtype=torch.float32) + + def __len__(self) -> int: + return self.X.shape[0] + + def __getitem__(self, idx: int): + return (self.X[idx], self.Y[idx], self.Y_mask[idx], self.Y_H[idx], self.Y_H_mask[idx], + self.forecast_precip[idx], self.forecast_missing[idx]) + + +def plot_selected_subgraph(basin_data: List[Tuple[pd.DataFrame, pd.DataFrame, int]], combined: Dict, output_path: Path) -> None: + """ + A real, static image of the actual selected subgraph -- which real + stations/confluences/braid points made it into training, and how + they're really connected -- saved once near the start of a run, so + a genuine visual sanity check (does this look like a real river + network, not a scrambled one?) is available without needing the + Streamlit explorer running at the same time. + + Real node type, from the same real columns used throughout this + project (is_gauged, is_confluence, is_split_point/is_rejoin_point), + not inferred or guessed -- confluences/braid points get their own + real, currently-known role marked, mirroring build_reach_graph.py's + own real type distinctions rather than treating every node the same. + """ + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(10, 10)) + colors = {"gauge": "#2563eb", "confluence": "#dc2626", "braid": "#f59e0b", "other": "#94a3b8"} + plotted_labels = set() + + for nodes_df, edges_df, basin_id in basin_data: + offset = combined["node_offsets"][basin_id] + lat = nodes_df["latitude"].values + lon = nodes_df["longitude"].values + + # Real edges, drawn first so node markers sit on top -- edge_index + # already has the real per-basin offset applied, matching this + # basin's own node ordering within the combined graph exactly. + edge_index = combined["edge_index"] + basin_node_range = range(offset, offset + len(nodes_df)) + for e in range(edge_index.shape[1]): + src, dst = edge_index[0, e], edge_index[1, e] + if src in basin_node_range and dst in basin_node_range: + src_local, dst_local = src - offset, dst - offset + ax.plot([lon[src_local], lon[dst_local]], [lat[src_local], lat[dst_local]], + color="#cbd5e1", linewidth=0.8, zorder=1) + + for node_type, mask in [ + ("gauge", nodes_df.get("is_gauged", pd.Series(False, index=nodes_df.index)).astype(bool)), + ("confluence", nodes_df.get("is_confluence", pd.Series(False, index=nodes_df.index)).astype(bool)), + ("braid", (nodes_df.get("is_split_point", pd.Series(False, index=nodes_df.index)).astype(bool) | + nodes_df.get("is_rejoin_point", pd.Series(False, index=nodes_df.index)).astype(bool))), + ]: + if mask.any(): + label = node_type if node_type not in plotted_labels else None + ax.scatter(lon[mask.values], lat[mask.values], c=colors[node_type], s=60, + label=label, zorder=3, edgecolors="white", linewidths=0.5) + plotted_labels.add(node_type) + + # "Other" -- real nodes present but not gauge/confluence/braid + # (shouldn't normally occur in this project's own subgraph + # selection, which explicitly keeps only those three types, but + # drawn explicitly rather than silently dropped if it ever does). + classified = (nodes_df.get("is_gauged", pd.Series(False, index=nodes_df.index)).astype(bool) | + nodes_df.get("is_confluence", pd.Series(False, index=nodes_df.index)).astype(bool) | + nodes_df.get("is_split_point", pd.Series(False, index=nodes_df.index)).astype(bool) | + nodes_df.get("is_rejoin_point", pd.Series(False, index=nodes_df.index)).astype(bool)) + other_mask = ~classified + if other_mask.any(): + label = "other" if "other" not in plotted_labels else None + ax.scatter(lon[other_mask.values], lat[other_mask.values], c=colors["other"], s=30, + label=label, zorder=2, alpha=0.6) + plotted_labels.add("other") + + ax.set_xlabel("Longitude") + ax.set_ylabel("Latitude") + ax.set_title(f"Selected training subgraph -- {combined['n_nodes']} real node(s), " + f"{combined['edge_index'].shape[1]} real edge(s)") + ax.legend(loc="best") + ax.set_aspect("equal", adjustable="datalim") + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def plot_training_curves(history: Dict[str, List[float]], output_path: Path) -> None: + """ + Real train/val loss curves, overwritten every epoch (not just once + at the end) -- since a training run here can genuinely take a long + time, this file can be opened at any point during a live run to + check real progress, not only after it completes or early-stops. + """ + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(10, 6)) + epochs = range(1, len(history["train_loss"]) + 1) + ax.plot(epochs, history["train_loss"], label="train loss", color="#2563eb", linewidth=1.5) + ax.plot(epochs, history["val_Q_loss"], label="val Q loss", color="#dc2626", linewidth=1.5) + ax.plot(epochs, history["val_H_loss"], label="val H loss", color="#f59e0b", linewidth=1.5) + ax.plot(epochs, history["val_combined"], label="val combined", color="#16a34a", linewidth=1.5, linestyle="--") + if history.get("best_epoch"): + ax.axvline(history["best_epoch"], color="#94a3b8", linestyle=":", linewidth=1, + label=f"best (epoch {history['best_epoch']})") + ax.set_xlabel("Epoch") + ax.set_ylabel("Loss") + ax.set_title("Training progress") + ax.legend(loc="best") + ax.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(output_path, dpi=150) + plt.close(fig) + + +def compute_empirical_quantiles(train_values_standardized: np.ndarray, quantiles: List[float]) -> np.ndarray: + """ + Real empirical quantiles from the training-period target + distribution (already standardized) -- the actual naive baseline + for quantile prediction. Predicting zero for every quantile level + would be wrong even as a naive baseline: the naive 90th-percentile + guess should be the training data's own real 90th percentile, not + the mean. This is the direct quantile-regression equivalent of + climatology in the earlier climate work -- the always-available + floor any real model needs to beat. + """ + real_values = train_values_standardized[~np.isnan(train_values_standardized)] + return np.array([np.percentile(real_values, q * 100) for q in quantiles]) + + +def compute_naive_baseline_loss( + val_loader: DataLoader, naive_quantiles: np.ndarray, quantiles: List[float], target: str = "Q", +) -> float: + """ + Pinball loss for the naive baseline (see compute_empirical_quantiles) + -- the SAME real training-period empirical quantile predicted for + every node and horizon, regardless of actual conditions. A real + model's validation loss needs to be compared against this directly, + the same discipline used throughout this project's climate work, + not just admired for being a small-looking number. + + target: "Q" (discharge) or "H" (water level) -- these have + genuinely different real coverage and distributions, so each needs + its own naive baseline computed from its own training data, not a + shared one. + """ + naive_quantiles_t = torch.tensor(naive_quantiles, dtype=torch.float32) + total, n = 0.0, 0 + for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, _, _ in val_loader: + target_batch = Y_batch if target == "Q" else Y_H_batch + for i in range(X_batch.shape[0]): + Y_i = target_batch[i] + naive_pred = naive_quantiles_t.view(1, 1, -1).expand(*Y_i.shape, len(quantiles)) + loss = multi_quantile_loss(Y_i, naive_pred, quantiles) + if torch.isfinite(loss): + total += loss.item() + n += 1 + return total / n if n > 0 else float("nan") + + +def train_spatiotemporal_model( + model: "SpatiotemporalGNN", train_loader: DataLoader, val_loader: DataLoader, + x_shared_static: "torch.Tensor", x_gauge_static: "torch.Tensor", is_gauged: "torch.Tensor", + edge_index: "torch.Tensor", edge_attr: "torch.Tensor", + basin_id: "torch.Tensor", horizons_t: "torch.Tensor", + confluence_index, braid_index, routing_index, physics_weights: Dict[str, float], + max_epochs: int, patience: int, lr: float, weight_decay: float, device: str, + precip_value_idx: Optional[int] = None, precip_missing_idx: Optional[int] = None, + catchment_area_km2_t: Optional["torch.Tensor"] = None, + discharge_mean: float = 0.0, discharge_std: float = 1.0, + precip_mean: float = 0.0, precip_std: float = 1.0, + plots_dir: Optional[Path] = None, + soft_dtw_gamma: float = 1.0, + discharge_per_node_mean: Optional["torch.Tensor"] = None, + waterlevel_per_node_mean: Optional["torch.Tensor"] = None, +) -> Dict: + """ + One real graph shared across every example in a batch -- looping + over individual examples within a batch (real, single-graph forward + passes) rather than building a replicated-graph mega-batch. Slower + per-step than a fully vectorized batched-graph approach would be, + but correct and simple, matching this project's established + practice of building the straightforward version first and only + adding complexity (like true graph batching) once the simple + version is shown to need it -- the same sequencing already applied + to the climate model's uncertainty head. + + Supervised loss uses ONLY the discharge target -- Y is genuinely 2D, + [n_nodes, n_horizons], with no target-type axis at all (confirmed + directly by the real printed shape Y=(4903, 4525, 10) from an actual + run -- an earlier version of this loop wrongly indexed Y_i[:, :, 0] + as if a target axis existed, which crashed immediately on real data + with "too many indices for tensor of dimension 2"). Model output + Q_pred is sliced down to the discharge channel (pred[:, :, 0]) to + match Y's real shape, not the other way around. Water-level targets + aren't wired into prepare_graph_training_windows yet (discharge-only + for this first build), so the model's water-level output channel + currently trains from physics losses alone. Worth revisiting once a + real water-level target tensor is added. + """ + model = model.to(device) + x_shared_static, x_gauge_static, is_gauged, edge_index, edge_attr, basin_id, horizons_t = ( + x_shared_static.to(device), x_gauge_static.to(device), is_gauged.to(device), + edge_index.to(device), edge_attr.to(device), basin_id.to(device), horizons_t.to(device) + ) + optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) + + best_val_loss = float("inf") + best_val_Q_loss = float("inf") + best_val_H_loss = float("inf") + best_state = None + epochs_without_improvement = 0 + # Real loss history, plotted (and re-plotted, overwriting the same + # file) at the end of every epoch -- not just once at the end of + # training -- so a genuinely long-running training job can be + # checked mid-run by opening the saved file, without needing to + # wait for completion or early stopping. + history = {"train_loss": [], "val_Q_loss": [], "val_H_loss": [], "val_combined": [], "best_epoch": None} + + debug_examples_remaining = 25 # widened from 3 -- the first 3 examples all looked fine, need more coverage to catch whatever's different about the examples that actually break + skipped_batches_to_report = 10 # report the actual batch_loss value for the first several skipped batches, not just that it happened + + for epoch in range(max_epochs): + model.train() + total_loss, n_batches = 0.0, 0 + for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch in train_loader: + optimizer.zero_grad() + batch_loss = X_batch.sum() * 0.0 # zero, correct dtype/device + for i in range(X_batch.shape[0]): + x_dynamic_seq = X_batch[i].to(device) + Y_i = Y_batch[i].to(device) + mask_i = mask_batch[i].to(device) + Y_H_i = Y_H_batch[i].to(device) + Y_H_mask_i = Y_H_mask_batch[i].to(device) + forecast_precip_i = forecast_precip_batch[i].to(device) + forecast_missing_i = forecast_missing_batch[i].to(device) + + pred = model(x_shared_static, x_gauge_static, x_dynamic_seq, is_gauged, edge_index, edge_attr, basin_id, horizons_t, + forecast_precip_i, forecast_missing_i) + # [n_nodes, n_horizons, n_quantiles] -- full quantile + # predictions per target, not a single point estimate. + Q_pred_quantiles = pred[:, :, 0, :] + H_pred_quantiles = pred[:, :, 1, :] + + # Pinball loss across every predicted quantile -- Y_i/Y_H_i + # are 2D [n_nodes, n_horizons] real targets, broadcast + # against each quantile prediction internally by + # multi_quantile_loss. mask_i/Y_H_mask_i aren't passed + # explicitly here: Y_i/Y_H_i already carry NaN at exactly + # the same positions the mask marks False (both derived + # from the same real target tensor), so pinball_loss's own + # NaN-masking already does the right thing without a + # separate mask argument. + supervised_Q = multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES) + supervised_H = multi_quantile_loss(Y_H_i, H_pred_quantiles, QUANTILES) + + # NSE: a real, different signal from pinball loss above -- + # checks whether the MEDIAN prediction tracks real + # variance, not whether the quantiles are well-calibrated. + # Applied to the median slice only (index 0, QUANTILES[0] + # = 0.5), matching every other physics-adjacent loss in + # this file. Weighted by physics_weights["nse"] alongside + # everything else, not treated as part of the supervised + # term itself, so it can be dialed independently. + nse_Q = Q_pred_quantiles.sum() * float("nan") + nse_H = H_pred_quantiles.sum() * float("nan") + if discharge_per_node_mean is not None: + nse_Q = nse_loss(Y_i, Q_pred_quantiles[:, :, 0], discharge_per_node_mean) + if waterlevel_per_node_mean is not None: + nse_H = nse_loss(Y_H_i, H_pred_quantiles[:, :, 0], waterlevel_per_node_mean) + + recent_precip_sum = None + if precip_value_idx is not None: + # Real, confirmed bug (not an instability, an actual + # physical mismatch): water_balance_loss is told + # period_days=ROUTING_HORIZON_COUNT, so the + # precipitation summed here MUST cover that same + # real period -- summing the full 30-day lookback + # instead compared 30 days of accumulated rain + # against a discharge volume representing only 5 + # days. For a real catchment as large as 5935 km2, + # that mismatch alone produced a residual squared to + # ~1e17 -- not instability, a genuinely wrong + # physical comparison, confirmed by hand-computing + # the exact real numbers from a real debug run. + precip_value_window = x_dynamic_seq[-ROUTING_HORIZON_COUNT:, :, precip_value_idx] # [ROUTING_HORIZON_COUNT, n_nodes] + precip_missing_window = x_dynamic_seq[-ROUTING_HORIZON_COUNT:, :, precip_missing_idx] + lookback_len = precip_value_window.shape[0] + real_fraction = 1.0 - precip_missing_window.mean(dim=0) # [n_nodes] + raw_sum_standardized = precip_value_window.sum(dim=0) # [n_nodes], STANDARDIZED units + + # Confirmed as a real bug against real training, not + # hypothetical: this sum was fed directly into + # water_balance_loss as if it were real mm, but + # x_dynamic_seq is built from the ALREADY-STANDARDIZED + # precipitation tensor -- a z-scored sum has no real + # physical meaning on its own and, combined with a + # real (unstandardized) catchment area, produced + # numbers with no physical basis at all. Un- + # standardizing a SUM of N days is not simply + # sum*std+mean -- the mean shift applies once PER + # REAL DAY summed, not once for the whole sum, so + # this uses the real count of real (non-missing) + # days in the window specifically, not the full + # lookback length. + n_real_days = real_fraction * lookback_len + real_precip_sum_mm = raw_sum_standardized * precip_std + n_real_days * precip_mean + + # A sum built from a mostly-missing window would + # silently understate real rainfall rather than + # honestly excluding it -- real coverage below 50% + # of the lookback window is treated as unreliable + # (NaN), letting water_balance_loss's own existing + # NaN-exclusion handle it correctly, the same + # completeness discipline already used for + # aggregate_anomalies in the climate work. + recent_precip_sum = torch.where(real_fraction >= 0.5, real_precip_sum_mm, + torch.full_like(real_precip_sum_mm, float("nan"))) + + should_debug = debug_examples_remaining > 0 + if should_debug: + debug_examples_remaining -= 1 + print(f" [debug] example (epoch {epoch+1}, remaining debug budget {debug_examples_remaining}):") + print(f" [debug] Q_pred_quantiles: finite_all={torch.isfinite(Q_pred_quantiles).all().item()}, " + f"min={Q_pred_quantiles.min().item():.4g}, max={Q_pred_quantiles.max().item():.4g}") + print(f" [debug] H_pred_quantiles: finite_all={torch.isfinite(H_pred_quantiles).all().item()}, " + f"min={H_pred_quantiles.min().item():.4g}, max={H_pred_quantiles.max().item():.4g}") + print(f" [debug] Y_i real value count: {int((~torch.isnan(Y_i)).sum())}/{Y_i.numel()}") + print(f" [debug] Y_H_i real value count: {int((~torch.isnan(Y_H_i)).sum())}/{Y_H_i.numel()}") + physics = compute_physics_loss(Q_pred_quantiles, H_pred_quantiles, HORIZONS, + confluence_index, braid_index, routing_index, + edge_index, edge_attr, model.manning_n_logit, model.log_bank_height_m, + physics_weights, recent_precip_sum, catchment_area_km2_t, + discharge_mean, discharge_std, soft_dtw_gamma=soft_dtw_gamma, + debug=should_debug) + if should_debug: + print(f" [debug] supervised_Q: {supervised_Q.item():.6g}, finite={torch.isfinite(supervised_Q).item()}") + print(f" [debug] supervised_H: {supervised_H.item():.6g}, finite={torch.isfinite(supervised_H).item()}") + print(f" [debug] nse_Q: {nse_Q.item():.6g}, finite={torch.isfinite(nse_Q).item()}") + print(f" [debug] nse_H: {nse_H.item():.6g}, finite={torch.isfinite(nse_H).item()}") + print(f" [debug] physics total: {physics.item():.6g}, finite={torch.isfinite(physics).item()}") + g_Q, = torch.autograd.grad(supervised_Q, [Q_pred_quantiles], retain_graph=True, allow_unused=True) + g_H, = torch.autograd.grad(supervised_H, [H_pred_quantiles], retain_graph=True, allow_unused=True) + g_Q_ok = g_Q is None or torch.isfinite(g_Q).all().item() + g_H_ok = g_H is None or torch.isfinite(g_H).all().item() + print(f" [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite={g_Q_ok}") + print(f" [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite={g_H_ok}") + if torch.isfinite(nse_Q): + g_nse_Q, = torch.autograd.grad(nse_Q, [Q_pred_quantiles], retain_graph=True, allow_unused=True) + g_nse_Q_ok = g_nse_Q is None or torch.isfinite(g_nse_Q).all().item() + print(f" [debug] GRADIENT of nse_Q: wrt Q_pred_quantiles finite={g_nse_Q_ok}") + if torch.isfinite(nse_H): + g_nse_H, = torch.autograd.grad(nse_H, [H_pred_quantiles], retain_graph=True, allow_unused=True) + g_nse_H_ok = g_nse_H is None or torch.isfinite(g_nse_H).all().item() + print(f" [debug] GRADIENT of nse_H: wrt H_pred_quantiles finite={g_nse_H_ok}") + + # Real water-level supervision, for the first time -- previously + # H_pred had zero direct targets, only manning_consistency_loss's + # one loose consistency constraint. Each term added only if + # finite (an anchor date can have real discharge coverage but + # no real water-level coverage in this specific window, or vice + # versa -- adding a NaN/inf term would poison the whole example). + example_loss = physics + if torch.isfinite(supervised_Q): + example_loss = example_loss + supervised_Q + if torch.isfinite(supervised_H): + example_loss = example_loss + supervised_H + if torch.isfinite(nse_Q): + example_loss = example_loss + physics_weights.get("nse", 1.0) * nse_Q + if torch.isfinite(nse_H): + example_loss = example_loss + physics_weights.get("nse", 1.0) * nse_H + batch_loss = batch_loss + example_loss + + batch_loss = batch_loss / X_batch.shape[0] + if not torch.isfinite(batch_loss): + if skipped_batches_to_report > 0: + skipped_batches_to_report -= 1 + print(f" [debug] SKIPPED a batch: batch_loss={batch_loss.item():.6g} " + f"(epoch {epoch+1}, batch_size={X_batch.shape[0]})") + continue + batch_loss.backward() + + # Real bug found and fixed here, not just a defensive + # addition: clip_grad_norm_ computes ONE combined norm + # across every parameter's gradient together. If even a + # single parameter's gradient is nan/inf, that poisons the + # TOTAL norm, and the resulting clip scale factor then gets + # multiplied into EVERY parameter's gradient -- silently + # spreading one corrupted gradient to the whole model on + # that step, rather than containing it. This is exactly + # what a real run showed: the Manning's roughness parameter + # itself became nan, + # and every forward pass afterward (training AND + # validation) produced non-finite output. Checking finiteness + # BEFORE clipping, and skipping the entire step if anything + # is already corrupted, is the actual fix -- clipping alone + # was solving the wrong layer of the problem. + grads_finite = all( + torch.isfinite(p.grad).all() for p in model.parameters() if p.grad is not None + ) + if not grads_finite: + if skipped_batches_to_report > 0: + skipped_batches_to_report -= 1 + print(f" [debug] SKIPPED at gradient-finiteness check (epoch {epoch+1}), " + f"batch_loss was {batch_loss.item():.6g} (finite) but a gradient was not:") + for name, p in model.named_parameters(): + if p.grad is not None and not torch.isfinite(p.grad).all(): + n_bad = (~torch.isfinite(p.grad)).sum().item() + print(f" [debug] non-finite gradient in '{name}': " + f"{n_bad}/{p.grad.numel()} bad entries, " + f"grad min={p.grad[torch.isfinite(p.grad)].min().item() if n_bad < p.grad.numel() else float('nan'):.4g}") + optimizer.zero_grad() + continue + + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0) + optimizer.step() + total_loss += batch_loss.item() + n_batches += 1 + + model.eval() + val_Q_total, val_Q_batches = 0.0, 0 + val_H_total, val_H_batches = 0.0, 0 + with torch.no_grad(): + for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch in val_loader: + for i in range(X_batch.shape[0]): + x_dynamic_seq = X_batch[i].to(device) + Y_i = Y_batch[i].to(device) + mask_i = mask_batch[i].to(device) + Y_H_i = Y_H_batch[i].to(device) + Y_H_mask_i = Y_H_mask_batch[i].to(device) + forecast_precip_i = forecast_precip_batch[i].to(device) + forecast_missing_i = forecast_missing_batch[i].to(device) + + pred = model(x_shared_static, x_gauge_static, x_dynamic_seq, is_gauged, edge_index, edge_attr, basin_id, horizons_t, + forecast_precip_i, forecast_missing_i) + Q_pred_quantiles = pred[:, :, 0, :] + H_pred_quantiles = pred[:, :, 1, :] + supervised_Q = multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES) + supervised_H = multi_quantile_loss(Y_H_i, H_pred_quantiles, QUANTILES) + # isfinite, not isnan -- confirmed as a real bug against + # real training: an untrained model produced an + # actually-infinite prediction, and isnan(inf) is False, + # so it passed this check unfiltered and poisoned the + # whole validation average to inf (observed directly: + # "val supervised loss = inf" on a real run). + if torch.isfinite(supervised_Q): + val_Q_total += supervised_Q.item() + val_Q_batches += 1 + if torch.isfinite(supervised_H): + val_H_total += supervised_H.item() + val_H_batches += 1 + + val_Q_loss = val_Q_total / val_Q_batches if val_Q_batches > 0 else float("inf") + val_H_loss = val_H_total / val_H_batches if val_H_batches > 0 else float("inf") + # Combined criterion for early stopping/model selection -- reported + # separately below too (not just this combined number), specifically + # so it's visible if one target is training well while the other + # isn't, rather than a single opaque average hiding that. + val_loss = (val_Q_loss if np.isfinite(val_Q_loss) else 0.0) + (val_H_loss if np.isfinite(val_H_loss) else 0.0) + if not np.isfinite(val_Q_loss) and not np.isfinite(val_H_loss): + val_loss = float("inf") + + print(f" [debug] raw accounting: n_batches={n_batches}, total_loss={total_loss:.6g}, " + f"len(train_loader)={len(train_loader)}") + print(f"epoch {epoch+1}/{max_epochs}: train loss = {total_loss/max(n_batches,1):.4f}, " + f"val Q loss = {val_Q_loss:.4f}, val H loss = {val_H_loss:.4f}, " + f"val combined = {val_loss:.4f}" + f"{' (best)' if val_loss < best_val_loss else ''}") + + history["train_loss"].append(total_loss / max(n_batches, 1)) + history["val_Q_loss"].append(val_Q_loss if np.isfinite(val_Q_loss) else float("nan")) + history["val_H_loss"].append(val_H_loss if np.isfinite(val_H_loss) else float("nan")) + history["val_combined"].append(val_loss if np.isfinite(val_loss) else float("nan")) + + if val_loss < best_val_loss: + best_val_loss = val_loss + best_val_Q_loss = val_Q_loss + best_val_H_loss = val_H_loss + best_state = {k: v.clone() for k, v in model.state_dict().items()} + epochs_without_improvement = 0 + history["best_epoch"] = epoch + 1 + else: + epochs_without_improvement += 1 + + if plots_dir is not None: + try: + plot_training_curves(history, plots_dir / "training_curves.png") + except Exception as e: + print(f" [warning] failed to save training curve plot this epoch: {e}") + + if epochs_without_improvement >= patience: + print(f"stopped early at epoch {epoch+1}") + break + + if best_state is not None: + model.load_state_dict(best_state) + return {"best_val_loss": best_val_loss, "best_val_Q_loss": best_val_Q_loss, + "best_val_H_loss": best_val_H_loss, "model_state": model.state_dict()} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Train the main spatiotemporal river GNN") + parser.add_argument("--data-root", type=Path, default=Path("datasets")) + parser.add_argument("--lookback-days", type=int, default=30) + parser.add_argument("--max-nodes", type=int, default=100, + help="Total node budget across both basins -- real gauges plus a capped number " + "of real confluences/braid points, no virtual infill nodes, for fast " + "training iteration. Real connectivity between kept nodes is preserved via " + "collapsed edges (see subgraph_selection.py), not simply dropped. Set to 0 " + "or a very large number to use the full real reach graph instead.") + parser.add_argument("--train-end", type=str, default="2021-12-31") + parser.add_argument("--train-start", type=str, default=None, + help="Iteration-speed lever for development, not a real training setting: " + "filters train/val to anchor dates on or after this date, for a fast " + "dev cycle while validating a code change. The full 2013+ dynamic " + "tensors are still built regardless (so lookback context isn't " + "truncated), only which anchor dates train against is reduced. Real " + "flood events are rare in this dataset -- a short window trades away " + "real signal for speed. Leave unset for a real, full training run.") + parser.add_argument("--val-end", type=str, default="2023-12-31") + parser.add_argument("--test-end", type=str, default="2026-12-31") + parser.add_argument("--confluence-weight", type=float, default=1.0) + parser.add_argument("--split-rejoin-weight", type=float, default=1.0) + parser.add_argument("--routing-weight", type=float, default=2.0, + help="Higher default than the others -- short-horizon routing is where " + "real travel-time physics is most directly checkable.") + parser.add_argument("--routing-soft-dtw-weight", type=float, default=1.0, + help="Weight for the soft-DTW routing shape-match term -- complements " + "--routing-weight's fixed-lag comparison with tolerance for the lag " + "estimate being slightly off.") + parser.add_argument("--routing-soft-dtw-gamma", type=float, default=0.001, + help="Soft-DTW smoothing. Controls a confirmed real failure mode: soft-DTW is " + "only guaranteed non-negative as gamma->0. The first fix (1.0->0.1) was " + "validated only against independent random sequences and missed that real " + "routing edges (upstream/downstream on the same river) are highly " + "correlated with small real differences -- a real training run then showed " + "this loss reading exactly 0 on every single example, floored by the " + "defensive clamp. gamma=0.001 showed 0/500 negative against both the " + "realistic correlated case and the original independent case. " + "physics_losses.py's routing_soft_dtw_loss also floors the result at 0 as " + "a second layer of protection, but don't rely on that alone -- validate any " + "change to this default against a correlated-sequence test.") + parser.add_argument("--manning-weight", type=float, default=1.0, + help="Weight for Manning's equation consistency between predicted discharge " + "and water level.") + parser.add_argument("--water-balance-weight", type=float, default=1.0, + help="Weight for water balance (precip - ET - Q - dS ~ 0), applied only at " + "the short contiguous-horizon block using real recent precipitation -- " + "see compute_physics_loss's docstring for why this can't be applied to " + "long horizons without real forecast precipitation. Automatically " + "disabled if no real precipitation data was fetched for this run.") + parser.add_argument("--nse-weight", type=float, default=1.0, + help="Weight for NSE (Nash-Sutcliffe Efficiency), a real, different signal " + "from pinball loss: checks whether the median prediction tracks each " + "station's own real historical variance, or flattens toward a safe " + "near-mean prediction regardless of quantile calibration elsewhere. " + "Applied to the median prediction only, same convention as every other " + "physics-adjacent loss in this file. Real per-node historical mean is " + "fit on the training period only, same discipline as discharge_mean/std.") + parser.add_argument("--batch-size", type=int, default=8, + help="Small default -- each example is a full graph forward pass looped " + "internally (see train_spatiotemporal_model's docstring), so this is " + "closer to 'graphs per gradient step' than a typical tabular batch size.") + parser.add_argument("--max-epochs", type=int, default=100) + parser.add_argument("--patience", type=int, default=10) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--weight-decay", type=float, default=1e-4) + args = parser.parse_args() + + shared_static_cols = ["elevation_m", "catchment_area_km2", "idpr_value", + "avg_groundwater_level_m", "distance_to_nearest_cavity_km", "n_cavities_within_20km"] + # Dropped raw latitude/longitude -- real risk of the model memorizing + # location-specific patterns at this project's real, small (100-node) + # subgraph scale rather than learning transferable hydrology; elevation, + # catchment area, and the graph's own topology already carry the + # spatial information that's actually physically load-bearing. + # Requires enrich_reach_graph.py to have been (re-)run with real + # cavity-proximity wiring -- shared_static_cols uses direct column + # selection, so a column genuinely absent from a basin's enriched + # CSV raises here rather than silently degrading. + gauge_static_cols: List[str] = [] + + max_nodes_per_basin = args.max_nodes // len(BASIN_FILE_NAMES) + basin_data = [] + for basin_id, file_key in BASIN_FILE_NAMES.items(): + nodes_df, edges_df = load_basin_graph(args.data_root, file_key) + print(f"{file_key}: {len(nodes_df)} real nodes, {len(edges_df)} real edges (full reach graph)") + + if args.max_nodes: + nodes_df, edges_df = build_collapsed_subgraph( + nodes_df, edges_df, max_nodes=max_nodes_per_basin, + ) + print(f"{file_key}: reduced to {len(nodes_df)} node(s) (real gauges + confluences/braids only, " + f"no virtual infill nodes), {len(edges_df)} collapsed edge(s), real connectivity preserved") + + basin_data.append((nodes_df, edges_df, basin_id)) + + combined = combine_basins(basin_data, shared_static_cols, gauge_static_cols) + print(f"Combined graph: {combined['n_nodes']} nodes, {combined['edge_index'].shape[1]} edges") + + output_dir = args.data_root / "spatiotemporal_gnn" + output_dir.mkdir(parents=True, exist_ok=True) + try: + plot_selected_subgraph(basin_data, combined, output_dir / "selected_subgraph.png") + print(f"Saved real subgraph visualization to {output_dir / 'selected_subgraph.png'}") + except Exception as e: + print(f" [warning] failed to save subgraph visualization: {e}") + + dynamic_tensors, target_tensor, waterlevel_target_tensor, dates = build_combined_dynamic_tensors( + combined, basin_data, args.data_root, ("2013-01-01", args.test_end) + ) + print(f"Real discharge coverage: {int((~np.isnan(target_tensor)).sum())} / {target_tensor.size}") + print(f"Real water-level coverage: {int((~np.isnan(waterlevel_target_tensor)).sum())} / {waterlevel_target_tensor.size}") + + # Real catchment area, needed here (BEFORE standardization) for the + # specific-discharge estimate below, and reused later as-is for the + # water-balance physics loss -- computed once, not twice. + catchment_area_km2 = extract_catchment_area_km2(basin_data, combined) + n_real_catchment = int((~np.isnan(catchment_area_km2)).sum()) + print(f"Real catchment area available for {n_real_catchment}/{combined['n_nodes']} node(s)") + + # A real, physically-grounded discharge ESTIMATE for stations with + # missing/no discharge data, built from real precipitation -- an + # INPUT feature, never a target. Fit ONLY on real (Q, precip, area) + # triples at gauged stations, training period only (see + # fit_specific_discharge_coefficient's docstring for why this is a + # standard hydrological technique, not a novel guess), then applied + # uniformly to every node -- including nodes with no real discharge + # at all, which is the actual point: they get a real, physically- + # motivated estimate to work with instead of nothing. + if "precipitation" in dynamic_tensors and n_real_catchment > 0: + try: + rolling_precip_sum = compute_rolling_precip_sum(dynamic_tensors["precipitation"], window_days=args.lookback_days) + specific_discharge_k = fit_specific_discharge_coefficient( + target_tensor, rolling_precip_sum, catchment_area_km2, dates, args.train_end, + ) + estimated_discharge = estimate_discharge_from_precip(rolling_precip_sum, catchment_area_km2, specific_discharge_k) + dynamic_tensors["estimated_discharge"] = estimated_discharge + n_real_estimates = int((~np.isnan(estimated_discharge)).sum()) + print(f"Real precipitation-based discharge estimate available for " + f"{n_real_estimates} node-date value(s) (input feature, never a target)") + except ValueError as e: + print(f"Skipping precipitation-based discharge estimate: {e}") + else: + missing_parts = [] + if "precipitation" not in dynamic_tensors: + missing_parts.append("real precipitation") + if n_real_catchment == 0: + missing_parts.append("real catchment area") + print(f"Skipping precipitation-based discharge estimate -- missing: {', '.join(missing_parts)}.") + + dynamic_tensors, target_tensor, waterlevel_target_tensor, standardization_stats = standardize_dynamic_tensors( + dynamic_tensors, target_tensor, waterlevel_target_tensor, dates, args.train_end, + optional_vars=["precipitation", "estimated_discharge"], + ) + for var, (mean, std) in standardization_stats.items(): + print(f"Standardized '{var}': train-period mean={mean:.2f}, std={std:.2f} " + f"(fit on real training-period values only)") + + # Physics indices need PER-BASIN node numbering with the SAME offset + # convention as combine_basins, so confluence/braid/routing indices + # computed on each basin's own nodes_df need the same offset applied. + combined_confluence_index, combined_braid_index, combined_routing_index = [], [], [] + for nodes_df, edges_df, basin_id in basin_data: + offset = combined["node_offsets"][basin_id] + conf_idx = build_confluence_index(nodes_df, edges_df) + combined_confluence_index.extend([(c + offset, [u + offset for u in ups]) for c, ups in conf_idx]) + braid_idx = build_braid_index(nodes_df) + combined_braid_index.extend([(s + offset, r + offset) for s, r in braid_idx]) + routing_idx = build_routing_index(nodes_df, edges_df) + combined_routing_index.extend([(u + offset, d + offset, lag) for u, d, lag in routing_idx]) + print(f"Physics indices: {len(combined_confluence_index)} confluences, " + f"{len(combined_braid_index)} braid pairs, {len(combined_routing_index)} routing edges") + + # Real days-since-last-real-observation, per dynamic variable -- + # computed from the SAME dynamic_tensors dict, over the FULL date + # range (not per-window -- see compute_days_since_last_real's own + # docstring for why that matters), then log-compressed before + # standardized values ever touch it. A binary missing flag alone + # can't tell a single-day sensor glitch from a months-long outage; + # this is the actual fix for that. + dt_tensors = { + var: compress_days_since_last_real(compute_days_since_last_real(tensor)) + for var, tensor in dynamic_tensors.items() + } + print(f"Real days-since-last-observation channel added for: {list(dt_tensors.keys())}") + + X, Y, Y_mask, anchor_dates = prepare_graph_training_windows( + combined["x_shared_static"], dynamic_tensors, target_tensor, dates, args.lookback_days, HORIZONS, + dt_tensors=dt_tensors, + ) + print(f"Training windows: X={X.shape}, Y={Y.shape}") + + Y_H = extract_targets_at_anchor_dates(waterlevel_target_tensor, dates, anchor_dates, HORIZONS) + Y_H_mask = ~np.isnan(Y_H) + n_real_wl = int(Y_H_mask.sum()) + print(f"Water-level targets at the same {len(anchor_dates)} anchor dates: Y_H={Y_H.shape}, " + f"{n_real_wl} real value(s)") + print() + + # Real forecasted precipitation, at the same anchor dates -- graceful + # degradation if the real forecast archive hasn't been fetched for + # this run: falls back to fully-missing tensors, matching exactly + # what the model's own forecast_precip=None default already does, + # so training still works correctly without it, just without this + # signal. + forecast_csv_path = args.data_root / "previous_runs_forecast.csv" + if forecast_csv_path.exists(): + nodes_df_combined = pd.DataFrame({"station_code": combined["station_codes"]}) + forecast_precip, forecast_missing = build_forecast_tensor( + nodes_df_combined, forecast_csv_path, anchor_dates, HORIZONS, + ) + n_real_forecast = int((forecast_missing == 0.0).sum()) + print(f"Real forecast precipitation available for {n_real_forecast} " + f"(example, node, horizon) value(s) -- real coverage is partial by design " + f"(only horizons {FORECAST_LEAD_TIMES}, only from 2024-03-01 onward)") + else: + forecast_precip = np.zeros((len(anchor_dates), combined["n_nodes"], len(HORIZONS)), dtype=np.float32) + forecast_missing = np.ones((len(anchor_dates), combined["n_nodes"], len(HORIZONS)), dtype=np.float32) + print("No real forecast archive found -- forecast precipitation fully missing for this run " + "(run fetch_previous_runs.py to enable it)") + print() + + + # Split by anchor date -- same real train/val/test separation + # principle already validated for the climate model, applied here to + # whichever anchor dates a given window actually starts from. + train_end_ts, val_end_ts = pd.Timestamp(args.train_end), pd.Timestamp(args.val_end) + anchor_dates_arr = pd.DatetimeIndex(anchor_dates) + train_mask = anchor_dates_arr <= train_end_ts + val_mask = (anchor_dates_arr > train_end_ts) & (anchor_dates_arr <= val_end_ts) + test_mask = anchor_dates_arr > val_end_ts + + if args.train_start: + # Iteration-speed lever, not a permanent training-data + # reduction: filters which anchor dates actually enter + # train/val, while dynamic tensors were still built from the + # real 2013-01-01 start (unchanged) -- so even an early + # kept anchor date's own lookback window still draws on real + # history before it, rather than being artificially truncated. + # Real flood events are rare in this dataset (125 discharge + # events across the whole 2013-2025 period) -- a short + # --train-start is a real, deliberate trade of some of that + # signal for faster iteration during development, not something + # to leave set for a final, real training run. + train_start_ts = pd.Timestamp(args.train_start) + train_mask = train_mask & (anchor_dates_arr >= train_start_ts) + val_mask = val_mask & (anchor_dates_arr >= train_start_ts) + print(f"--train-start {args.train_start} applied: filtering train/val to anchor dates on or after " + f"this date. Test split is unaffected. This is an iteration-speed lever -- rerun without " + f"--train-start for a real, full training run before trusting results.") + + print(f"Train examples: {train_mask.sum()}, val: {val_mask.sum()}, test: {test_mask.sum()}") + if train_mask.sum() == 0 or val_mask.sum() == 0: + print("Not enough examples in train or val split -- check --train-end/--val-end against the " + "real date range this data actually covers.") + return + + train_ds = GraphWindowDataset(X[train_mask], Y[train_mask], Y_mask[train_mask], Y_H[train_mask], Y_H_mask[train_mask], + forecast_precip[train_mask], forecast_missing[train_mask]) + val_ds = GraphWindowDataset(X[val_mask], Y[val_mask], Y_mask[val_mask], Y_H[val_mask], Y_H_mask[val_mask], + forecast_precip[val_mask], forecast_missing[val_mask]) + train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False) + + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"Using device: {device}") + + x_shared_static_std, shared_static_stats, shared_static_flagged = standardize_static_features( + combined["x_shared_static"], shared_static_cols, + ) + x_gauge_static_std, gauge_static_stats, gauge_static_flagged = standardize_static_features( + combined["x_gauge_static"], gauge_static_cols, + ) + print(f"Static feature standardization: shared={shared_static_cols} " + f"(missingness flag added for: {shared_static_flagged or 'none'}), " + f"gauge={gauge_static_cols} (missingness flag added for: {gauge_static_flagged or 'none'})") + x_shared_static_t = torch.tensor(x_shared_static_std, dtype=torch.float32) + x_gauge_static_t = torch.tensor(x_gauge_static_std, dtype=torch.float32) + is_gauged_t = torch.tensor(combined["is_gauged"], dtype=torch.bool) + edge_index_t = torch.tensor(combined["edge_index"], dtype=torch.long) + edge_attr_t = torch.tensor(combined["edge_attr"], dtype=torch.float32) + basin_id_t = torch.tensor(combined["basin_id"], dtype=torch.long) + horizons_t = torch.tensor(HORIZONS, dtype=torch.float32) + + n_gauge_dynamic_channels = X.shape[-1] # value+missingness per dynamic variable, already interleaved + model = SpatiotemporalGNN( + shared_static_dim=x_shared_static_t.shape[1], gauge_static_dim=x_gauge_static_t.shape[1], + gauge_dynamic_dim=n_gauge_dynamic_channels, edge_dim=edge_attr_t.shape[1], + n_basins=len(BASIN_FILE_NAMES), n_targets=2, # [discharge, water_level] + quantiles=QUANTILES, + ) + + physics_weights = {"confluence": args.confluence_weight, "split_rejoin": args.split_rejoin_weight, + "routing": args.routing_weight, "manning": args.manning_weight, + "water_balance": args.water_balance_weight, + "routing_soft_dtw": args.routing_soft_dtw_weight, + "nse": args.nse_weight} + print(f"Physics loss weights: {physics_weights}") + print() + + # Real precipitation channel index -- None if the real precipitation + # CSV wasn't fetched for this run, in which case water balance is + # skipped entirely inside compute_physics_loss, not silently faked. + precip_value_idx, precip_missing_idx = None, None + if "precipitation" in dynamic_tensors: + var_names_for_channels = list(dynamic_tensors.keys()) + precip_value_idx, precip_missing_idx = get_dynamic_channel_index( + var_names_for_channels, "precipitation", channels_per_var=3, + ) + print(f"Real precipitation available -- water balance physics enabled " + f"(channels {precip_value_idx}/{precip_missing_idx})") + else: + print("No real precipitation data for this run -- water balance physics disabled " + "(run fetch_meteofrance_precipitation.py to enable it)") + + catchment_area_km2_t = torch.tensor(catchment_area_km2, dtype=torch.float32) + + discharge_mean, discharge_std = standardization_stats["discharge"] + precip_mean, precip_std = standardization_stats.get("precipitation", (0.0, 1.0)) + + # Real per-node historical mean, for NSE -- fit on the training + # period only (same discipline as discharge_mean/std above), from + # the already-standardized target_tensor so it's in the same real + # units the supervised loss and the model's own predictions + # actually operate in. + train_end_ts_for_nse = pd.Timestamp(args.train_end) + train_date_mask = np.array([d <= train_end_ts_for_nse for d in dates]) + discharge_per_node_mean_t = torch.tensor( + compute_per_node_historical_mean(target_tensor[:, train_date_mask]), dtype=torch.float32, + ) + waterlevel_per_node_mean_t = torch.tensor( + compute_per_node_historical_mean(waterlevel_target_tensor[:, train_date_mask]), dtype=torch.float32, + ) + + result = train_spatiotemporal_model( + model, train_loader, val_loader, x_shared_static_t, x_gauge_static_t, is_gauged_t, + edge_index_t, edge_attr_t, basin_id_t, horizons_t, + combined_confluence_index, combined_braid_index, combined_routing_index, physics_weights, + args.max_epochs, args.patience, args.lr, args.weight_decay, device, + precip_value_idx, precip_missing_idx, catchment_area_km2_t, + discharge_mean, discharge_std, precip_mean, precip_std, + plots_dir=output_dir, + soft_dtw_gamma=args.routing_soft_dtw_gamma, + discharge_per_node_mean=discharge_per_node_mean_t, + waterlevel_per_node_mean=waterlevel_per_node_mean_t, + ) + + model_path = output_dir / "model.pt" + if model_path.exists(): + # Archive the previous checkpoint before overwriting it -- this + # project has cycled through many real checkpoints this session + # (Manning's reparameterization, HORIZONS narrowing, NSE weight + # tuning, ...), each incompatible with the last in a real, + # structural way (not just "worse weights") -- losing the + # ability to go back to a real, previously-evaluated checkpoint + # has been a real, avoidable risk, not a hypothetical one. + archive_dir = output_dir / "archive" + archive_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + archived_path = archive_dir / f"model_{timestamp}.pt" + shutil.move(str(model_path), str(archived_path)) + print(f"Archived previous checkpoint to {archived_path}") + torch.save(result["model_state"], model_path) + print(f"\nBest validation loss: {result['best_val_loss']:.4f}") + learned_n = 0.01 + 0.29 * torch.sigmoid(model.manning_n_logit).item() + print(f"Learned Manning's roughness n = {learned_n:.4f} " + f"(0.03 was the initialization, real natural streams typically fall in ~0.025-0.1; " + f"now sigmoid-bounded to [0.01, 0.3] by construction, so it can no longer drift outside " + f"that range the way it did before -- but landing at or very near either boundary " + f"(0.01 or 0.3) still means it saturated rather than converged to a real interior value, " + f"and is still worth investigating, not just noting)") + learned_bank_height = torch.exp(model.log_bank_height_m).item() + print(f"Learned bank height = {learned_bank_height:.2f}m " + f"(2.5m was the initialization, an assumption with no real measured data behind it -- " + f"compare this against real rating-curve data for the gauges with one, if the model's " + f"learned value looks implausible for these specific rivers)") + print(f"Saved model to {model_path}") + + # Real empirical quantiles from the TRAINING period specifically -- + # same "fit on train only" discipline as standardization itself, so + # this naive baseline can't leak val/test information either. + train_col_mask = pd.DatetimeIndex(dates) <= pd.Timestamp(args.train_end) + naive_Q_quantiles = compute_empirical_quantiles(target_tensor[:, train_col_mask], QUANTILES) + naive_H_quantiles = compute_empirical_quantiles(waterlevel_target_tensor[:, train_col_mask], QUANTILES) + + naive_baseline_Q = compute_naive_baseline_loss(val_loader, naive_Q_quantiles, QUANTILES, target="Q") + naive_baseline_H = compute_naive_baseline_loss(val_loader, naive_H_quantiles, QUANTILES, target="H") + print(f"\nNaive baseline, discharge (real training-period empirical quantiles {QUANTILES}): {naive_baseline_Q:.4f}") + print(f"Naive baseline, water level (real training-period empirical quantiles {QUANTILES}): {naive_baseline_H:.4f}") + + for label, model_loss, baseline in [ + ("Discharge", result["best_val_Q_loss"], naive_baseline_Q), + ("Water level", result["best_val_H_loss"], naive_baseline_H), + ]: + if model_loss < baseline: + improvement_pct = 100 * (baseline - model_loss) / baseline + print(f"{label}: model beats the naive baseline by {improvement_pct:.1f}% -- real, not just baseline-matching skill") + else: + print(f"{label}: model does NOT beat the naive baseline -- has not yet demonstrated real skill " + f"beyond predicting the mean for this target specifically.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/data/loaders/__pycache__/safran.cpython-311.pyc b/src/data/loaders/__pycache__/safran.cpython-311.pyc index 1437a89b59bd7361a6d59c0a71a78bbb4a25ca11..be595527ce591933aece89a940c964bc55a086ee 100644 Binary files a/src/data/loaders/__pycache__/safran.cpython-311.pyc and b/src/data/loaders/__pycache__/safran.cpython-311.pyc differ diff --git a/src/data/loaders/safran.py b/src/data/loaders/safran.py index 7aba30d98f9afa57c8434ddaa2c9d94598e1c145..4aaec8a8e09b4d59ee783ea4ef54c9c034435326 100644 --- a/src/data/loaders/safran.py +++ b/src/data/loaders/safran.py @@ -73,6 +73,86 @@ class SAFRANLoader(BaseDataLoader): combined = pd.concat(all_dfs, ignore_index=True) return combined.sort_values(['date', 'station_code']).reset_index(drop=True) + def load_grid(self, bbox: Optional[tuple] = None) -> pd.DataFrame: + """ + Load ERA5 at its NATIVE grid resolution (0.25 degrees) -- no + per-station interpolation at all, unlike load(). Confirmed + against real data that station-level interpolation duplicates + signal: multiple real stations landed on the exact identical + grid cell (several stations sharing bit-identical wind_speed_ms + values across the whole real climatology test), since ERA5's + 0.25 deg cells are coarse relative to how tightly some real + gauges cluster. Modeling at grid-cell resolution avoids this + redundancy directly, rather than training separate models on + what's provably the same input. + + Args: + bbox: optional (min_lon, min_lat, max_lon, max_lat) to + subset the grid before loading -- the whole point is to + avoid loading the full national/continental ERA5 extent + when only a couple dozen cells actually cover the study + area. Defaults to no subsetting (the full extent + already present in the downloaded files, which + download_era5_full.py already bounded at fetch time). + + Returns: + DataFrame with [date, grid_lat, grid_lon, temp_2m_K, + precip_m, ...] -- one row per (date, cell), long format + (not yet pivoted to wide per-variable, since the number of + variables and their names come from VAR_MAP and shouldn't + be hardcoded here). + """ + nc_files = sorted(Path(self.data_path).glob("era5_*.nc")) + if not nc_files: + raise FileNotFoundError(f"No ERA5 files in {self.data_path}") + + all_dfs = [] + for nc_file in nc_files: + df = self._load_grid_single_file(nc_file, bbox) + all_dfs.append(df) + + combined = pd.concat(all_dfs, ignore_index=True) + return combined.sort_values(['date', 'grid_lat', 'grid_lon']).reset_index(drop=True) + + def _load_grid_single_file(self, nc_file: Path, bbox: Optional[tuple]) -> pd.DataFrame: + """Load a single NetCDF file's full grid (optionally bbox-subset), + no station interpolation. Every (lat, lon, date) triple becomes + one row -- this is real gridded data, not point samples.""" + ds = xr.open_dataset(nc_file) + time_coord = 'valid_time' if 'valid_time' in ds.coords else 'time' + + if bbox is not None: + min_lon, min_lat, max_lon, max_lat = bbox + # ERA5 latitude is typically stored descending (90 to -90); + # slice() requires matching that order or it silently + # returns nothing, which is exactly the kind of thing that + # produced a wrong-axis-order failure earlier in this + # project (BD TOPO's WFS) -- check the actual coordinate + # direction rather than assume one. + lat_vals = ds['latitude'].values + if lat_vals[0] > lat_vals[-1]: + ds = ds.sel(latitude=slice(max_lat, min_lat), longitude=slice(min_lon, max_lon)) + else: + ds = ds.sel(latitude=slice(min_lat, max_lat), longitude=slice(min_lon, max_lon)) + if ds.sizes.get('latitude', 0) == 0 or ds.sizes.get('longitude', 0) == 0: + raise ValueError( + f"bbox {bbox} selected zero grid cells from {nc_file.name} -- check the bbox " + f"is within this file's real coordinate range: " + f"lat {ds['latitude'].values.min() if ds.sizes.get('latitude',1) else 'N/A'}.." + f"{ds['latitude'].values.max() if ds.sizes.get('latitude',1) else 'N/A'}, " + f"lon similarly." + ) + + df = ds.to_dataframe().reset_index() + df = df.rename(columns={time_coord: 'date', 'latitude': 'grid_lat', 'longitude': 'grid_lon', **self.VAR_MAP}) + + keep_cols = ['date', 'grid_lat', 'grid_lon'] + [v for v in self.VAR_MAP.values() if v in df.columns] + df = df[keep_cols].copy() + df['date'] = pd.to_datetime(df['date']).dt.tz_localize(None) + + ds.close() + return df + def _load_and_interpolate(self, nc_file: Path) -> pd.DataFrame: """ Load single NetCDF and interpolate to ALL stations in one @@ -95,14 +175,11 @@ class SAFRANLoader(BaseDataLoader): latitude/longitude dimensions. This turns ~193,000 calls into ~67 (one per file). - CAVEAT -- I could not actually run this against xarray/netCDF4 - in this environment (no network access to install them here), - so this is reasoned from documented xarray API behavior, not - verified execution like the rest of this project's fixes have - been. Please test carefully, ideally against a small subset of - the era5_*.nc files first, and confirm the output matches the - old per-station-loop version's shape/values before trusting it - for real training data. + Reasoned from documented xarray API behavior, not verified + against a real NetCDF file with real execution -- test against + a small subset of the era5_*.nc files first, and confirm the + output matches the old per-station-loop version's shape/values + before trusting it for real training data. """ ds = xr.open_dataset(nc_file) time_coord = 'valid_time' if 'valid_time' in ds.coords else 'time' diff --git a/src/graph/__pycache__/climate_climatology.cpython-311.pyc b/src/graph/__pycache__/climate_climatology.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d115a6f6989552f763f62d50ca20a3650312c031 Binary files /dev/null and b/src/graph/__pycache__/climate_climatology.cpython-311.pyc differ diff --git a/src/graph/__pycache__/dynamic_features.cpython-311.pyc b/src/graph/__pycache__/dynamic_features.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dc9215a359b1c6a8c5efaebaf71eb9bdf614542 Binary files /dev/null and b/src/graph/__pycache__/dynamic_features.cpython-311.pyc differ diff --git a/src/graph/__pycache__/node_features.cpython-311.pyc b/src/graph/__pycache__/node_features.cpython-311.pyc index f07e795a20e7d15e9fd3f55a31f66a0fe2f8efd8..7d5c80787372c2e5cd37efe538d28871af34d2e8 100644 Binary files a/src/graph/__pycache__/node_features.cpython-311.pyc and b/src/graph/__pycache__/node_features.cpython-311.pyc differ diff --git a/src/graph/__pycache__/physics_losses.cpython-311.pyc b/src/graph/__pycache__/physics_losses.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2341226194e60e7f2e92346373044d3cb5e43b5c Binary files /dev/null and b/src/graph/__pycache__/physics_losses.cpython-311.pyc differ diff --git a/src/graph/__pycache__/pinball_loss.cpython-311.pyc b/src/graph/__pycache__/pinball_loss.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93635ea98fc5cdb736d79fb1dde3743a9c8e3eb4 Binary files /dev/null and b/src/graph/__pycache__/pinball_loss.cpython-311.pyc differ diff --git a/src/graph/__pycache__/spatiotemporal_gnn.cpython-311.pyc b/src/graph/__pycache__/spatiotemporal_gnn.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f949c7c8bc421ea17a22a78b59603386ce42008d Binary files /dev/null and b/src/graph/__pycache__/spatiotemporal_gnn.cpython-311.pyc differ diff --git a/src/graph/__pycache__/subgraph_selection.cpython-311.pyc b/src/graph/__pycache__/subgraph_selection.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4340cdd4e5b05dcf294bba0fcc4d82ccc02607c Binary files /dev/null and b/src/graph/__pycache__/subgraph_selection.cpython-311.pyc differ diff --git a/src/graph/climate_anomaly_model.py b/src/graph/climate_anomaly_model.py new file mode 100644 index 0000000000000000000000000000000000000000..580206184a54168816b922bdba787bc37e4e2c43 --- /dev/null +++ b/src/graph/climate_anomaly_model.py @@ -0,0 +1,264 @@ +""" +Anomaly model for the climate sub-model: predicts (actual - climatology) +at a requested horizon, given a recent history of anomalies. Combined +with climate_climatology.py's climatology at inference time: + + forecast(date, horizon) = climatology(day_of_year) + predicted_anomaly(horizon) + +DESIGN CHOICE, stated directly rather than left implicit: the predicted +anomaly is passed through a learnable, per-variable, MONOTONICALLY +DECAYING gate as a function of horizon -- built in architecturally, not +left for training to discover on its own. This is a direct response to +what the real climatology results showed: precipitation has close to +zero seasonal predictability even at the baseline level (4/20 real grid +cells beat a flat mean, and the margins were negligible), so there is +no reason to expect a data-driven model to have genuine multi-month +signal for it either. Rather than risk the model outputting a +confident-looking but meaningless anomaly at day 180, the gate forces +the anomaly's contribution toward zero as horizon grows -- how fast is +LEARNED per variable (a temperature-like variable with real persistence +should decay slowly; precipitation should decay fast), but the +DIRECTION (toward zero, never away from it) is guaranteed by +construction, not hoped for. + +Verification status: prepare_training_windows and decay_gate are pure +numpy/pandas, tested directly below. The nn.Module (ClimateAnomalyModel) +has been syntax-checked but not run end-to-end -- run +scripts/test_climate_anomaly_model.py against real data and confirm +behavior before trusting it, the same way the climatology and +grid-extraction work was validated. +""" +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd + +try: + import torch + import torch.nn as nn + _HAS_TORCH = True +except ImportError: + _HAS_TORCH = False + nn = None # only referenced inside the class body below, never at import time otherwise + + +# --------------------------------------------------------------------------- +# Data preparation -- pure numpy/pandas, fully testable without torch +# --------------------------------------------------------------------------- + +def prepare_training_windows( + anomaly_wide: pd.DataFrame, lookback_days: int, horizons: List[int], +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[pd.Timestamp]]: + """ + Slices a [date x cell] anomaly DataFrame into (history, target) + training examples: for every date with a full lookback window + behind it AND real target values at every requested horizon ahead + of it, extract the lookback window as input and the horizon-ahead + anomalies as targets. + + A window is only kept if the ENTIRE lookback period is real + (non-NaN) for that cell -- a history with gaps silently interpolated + would teach the model on fabricated input, not real conditions. A + target is NaN-masked per (example, horizon) individually rather + than requiring every horizon to be real, since discharge/climate + series have real, uneven gaps and requiring all horizons real would + throw away most of the sparse real data this project already has. + + Args: + anomaly_wide: [date x cell_id], from climate_climatology. + compute_anomalies. + lookback_days: how much history feeds the encoder. + horizons: which day-ahead offsets to produce targets for, e.g. + [1, 5, 10, 30, 60, 90, 180]. + + Returns: + X: [n_examples, lookback_days, n_cells] -- history window, + same set of cells for every example (aligned columns). + Y: [n_examples, n_horizons] -- target anomaly at each horizon, + for the SPECIFIC cell/column indicated by `example_cell_idx`. + example_cell_idx: [n_examples] -- which column (cell) index in + X's last dimension this example's Y targets refer to (one + training example = one cell's history predicting that SAME + cell's own future, not cross-cell). + dates: the anchor date (the "today") for each example, for + traceability. + """ + max_horizon = max(horizons) + values = anomaly_wide.values + dates = anomaly_wide.index + n_dates, n_cells = values.shape + + X_list, Y_list, cell_idx_list, anchor_dates = [], [], [], [] + + for t in range(lookback_days, n_dates - max_horizon): + window = values[t - lookback_days:t, :] + window_complete_mask = ~np.isnan(window).any(axis=0) # per cell + if not window_complete_mask.any(): + continue + + for cell in np.where(window_complete_mask)[0]: + targets = np.array([values[t + h - 1, cell] for h in horizons]) + if np.isnan(targets).all(): + continue + X_list.append(window[:, cell]) + Y_list.append(targets) + cell_idx_list.append(cell) + anchor_dates.append(dates[t]) + + if not X_list: + return (np.empty((0, lookback_days)), np.empty((0, len(horizons))), + np.empty((0,), dtype=int), []) + + X = np.stack(X_list)[:, :, None] # [n_examples, lookback_days, 1] -- single-variable per call + Y = np.stack(Y_list) + return X, Y, np.array(cell_idx_list), anchor_dates + + +def decay_gate(horizons: np.ndarray, tau: float) -> np.ndarray: + """ + exp(-h / tau) -- the monotonic decay curve gating the predicted + anomaly toward zero as horizon grows. tau is learned per variable + in the real model (see ClimateAnomalyModel); this standalone + version is what's actually tested here, since the shape/behavior + of the curve is the real design claim, independent of whether it's + a fixed constant or a learned nn.Parameter wrapping this same formula. + + tau is the horizon (in days) at which the gate has decayed to 1/e + (~37%) of its starting value -- a small tau means fast decay + (appropriate for precipitation, per the real climatology results), + a large tau means slow decay (appropriate for temperature). + """ + return np.exp(-horizons / tau) + + +# --------------------------------------------------------------------------- +# Model. Forward-pass shape/behavior verified only via the standalone +# decay-gate check above; the full nn.Module has not been independently +# run end-to-end. +# --------------------------------------------------------------------------- + +if _HAS_TORCH: + class ClimateAnomalyModel(nn.Module): + """ + Encoder: GRU over the lookback window of anomalies -> final + hidden state per example. + Decoder: [hidden_state, horizon_embedding] -> MLP -> (mean, + log_variance) per horizon. + + Two complementary mechanisms handle "the model shouldn't be + confidently wrong when there's no real signal" -- confirmed as + a real, not hypothetical, problem: the first version of this + model (mean-only, plain MSE) was measurably WORSE than pure + climatology at several horizons for temp_C/solar_Wm2/ + wind_speed_ms, while a persistence baseline diagnostic showed + there was no real short-horizon signal for those variables to + find in the first place -- the model was outputting confident, + wrong guesses instead of admitting it didn't know. + + 1. The MEAN is passed through a learnable, monotonically + decaying gate per variable (see decay_gate) -- a HARD + architectural constraint: the model is structurally unable + to output a large mean anomaly at a horizon its own learned + tau says is past the point of real signal. Proven to work + correctly on real data already: trained on snow_mm (a + genuinely close-to-unpredictable variable), tau collapsed to + 2.3 and the model's predictions converged to matching + climatology almost exactly at every horizon beyond a few days. + 2. The VARIANCE is predicted separately, NOT gated the same + way, and trained via Gaussian NLL loss rather than plain + MSE. This is what actually targets the diagnosed + overconfidence problem: NLL rewards the model for outputting + high predicted variance when it genuinely can't predict + well, rather than being forced into a single point guess + the loss then punishes regardless of whether real signal + existed to make a better one. + + One model per climate variable, not a shared model across all + nine -- the real climatology and persistence-diagnostic results + showed genuinely different predictability profiles (runoff_mm + beat climatology at every horizon; wind_v_ms/precip_mm/snow_mm + essentially never did), and forcing one shared decay/variance + behavior to serve both would misrepresent at least one of them. + """ + + def __init__(self, hidden_dim: int = 32, horizon_embed_dim: int = 8, init_tau: float = 30.0): + super().__init__() + self.encoder = nn.GRU(input_size=1, hidden_size=hidden_dim, batch_first=True) + self.horizon_embed = nn.Linear(1, horizon_embed_dim) + self.decoder = nn.Sequential( + nn.Linear(hidden_dim + horizon_embed_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 2), # [mean, log_variance] per horizon + ) + # log_tau, not tau directly, so tau = exp(log_tau) is always + # positive regardless of what gradient descent does to the + # underlying parameter -- a negative or zero tau would make + # the decay gate nonsensical (blow up or divide by zero). + self.log_tau = nn.Parameter(torch.log(torch.tensor(float(init_tau)))) + + def forward(self, history: "torch.Tensor", horizons: "torch.Tensor") -> Tuple["torch.Tensor", "torch.Tensor"]: + """ + Args: + history: [batch, lookback_days, 1] -- real anomaly + history, already NaN-free (prepare_training_windows + only emits complete windows). + horizons: [n_horizons] -- day-ahead offsets to predict + for, shared across the batch. + + Returns: + (mean, log_variance), each [batch, n_horizons]. The mean + is already gated by the learned decay curve; log_variance + is the model's own learned, per-example uncertainty, + deliberately not put through the same gate -- a model + should be able to express high uncertainty at ANY + horizon, including short ones, if a specific example + genuinely warrants it, not only according to a fixed + schedule. + """ + _, h_n = self.encoder(history) + hidden = h_n.squeeze(0) # [batch, hidden_dim] + + batch_size = hidden.shape[0] + n_horizons = horizons.shape[0] + hidden_expanded = hidden.unsqueeze(1).expand(batch_size, n_horizons, hidden.shape[-1]) + horizon_feat = self.horizon_embed(horizons.float().view(1, n_horizons, 1).expand(batch_size, n_horizons, 1)) + + decoder_input = torch.cat([hidden_expanded, horizon_feat], dim=-1) + raw_output = self.decoder(decoder_input) # [batch, n_horizons, 2] + raw_mean = raw_output[..., 0] + log_var = raw_output[..., 1] + + tau = torch.exp(self.log_tau) + gate = torch.exp(-horizons.float() / tau) # [n_horizons] + mean = raw_mean * gate.unsqueeze(0) + + return mean, log_var + + + def gaussian_nll_loss(mean: "torch.Tensor", log_var: "torch.Tensor", target: "torch.Tensor", + min_log_var: float = -10.0) -> "torch.Tensor": + """ + Heteroscedastic Gaussian negative log-likelihood, NaN-masked + (same masking principle used throughout this project's physics + losses -- a target that doesn't exist for a given [example, + horizon] pair must not silently poison every other real pair's + gradient the same way an un-masked MSE mean would). + + NLL = 0.5 * log_var + 0.5 * (target - mean)^2 / exp(log_var) + + min_log_var clamps how confident the model is allowed to claim + to be -- without a floor, a model can trivially drive NLL toward + -infinity by predicting an extremely small variance on an + example it happens to get right by chance, which is a real, + well-known failure mode of unconstrained heteroscedastic + regression, not a hypothetical one. + """ + mask = ~torch.isnan(target) + if not mask.any(): + return mean.sum() * float("nan") + log_var_clamped = torch.clamp(log_var[mask], min=min_log_var) + residual_sq = (target[mask] - mean[mask]) ** 2 + return (0.5 * log_var_clamped + 0.5 * residual_sq / torch.exp(log_var_clamped)).mean() +else: + ClimateAnomalyModel = None # torch not installed; import guard, not a silent no-op + gaussian_nll_loss = None \ No newline at end of file diff --git a/src/graph/climate_climatology.py b/src/graph/climate_climatology.py new file mode 100644 index 0000000000000000000000000000000000000000..99947ebd51fed6a00d8eeb401623be4ff4108074 --- /dev/null +++ b/src/graph/climate_climatology.py @@ -0,0 +1,207 @@ +""" +Harmonic-regression climatology: the honest, always-available floor for +the climate sub-model's forecast (see the model-design discussion -- +this exists specifically so the anomaly/deep model has something real +to be evaluated against, and something to gracefully degrade toward at +long lead times rather than producing an unstable raw extrapolation). + +y(day_of_year) = mean + sum_k [a_k*cos(2*pi*k*doy/365.25) + b_k*sin(2*pi*k*doy/365.25)] + +Plain least-squares on a fixed sin/cos basis -- no ML library needed, +genuinely testable with real data (climatology as a technique applies +to any seasonal series, not just climate variables specifically; tested +here against real discharge, which has its own real seasonal cycle, as +a stand-in until real ERA5 files are available to validate the actual +climate use case directly). +""" +from typing import Dict, Optional + +import numpy as np +import pandas as pd + + +def _day_of_year_fraction(dates: pd.DatetimeIndex) -> np.ndarray: + """Day-of-year as a fraction of 365.25, handling leap years by using + the actual day-of-year rather than a fixed 365-day assumption.""" + return dates.dayofyear.values / 365.25 + + +def _harmonic_basis(doy_frac: np.ndarray, n_harmonics: int) -> np.ndarray: + """[1, cos(2*pi*doy), sin(2*pi*doy), cos(4*pi*doy), sin(4*pi*doy), ...]""" + cols = [np.ones_like(doy_frac)] + for k in range(1, n_harmonics + 1): + cols.append(np.cos(2 * np.pi * k * doy_frac)) + cols.append(np.sin(2 * np.pi * k * doy_frac)) + return np.column_stack(cols) + + +def fit_harmonic_climatology( + wide_df: pd.DataFrame, n_harmonics: int = 2, min_months_covered: int = 8, +) -> Dict[str, np.ndarray]: + """ + Fits a smooth climatological baseline per column (station or grid + cell -- this function doesn't care which, it just needs a + [date-indexed] wide DataFrame) via least-squares harmonic + regression on day-of-year. + + n_harmonics=2 gives a reasonably flexible seasonal shape (captures + e.g. a spring peak that isn't a pure single sine wave) without + overfitting to short-term noise -- day-of-year alone, with no + dependence on which specific year, is what makes this genuinely a + CLIMATOLOGY and not just a smoothed history: it can predict any + future date, arbitrarily far out, since it only needs day-of-year. + + Requires real data spanning at least `min_months_covered` distinct + calendar months, not just enough raw POINTS -- confirmed as a real, + not hypothetical, failure mode against real discharge data: a + station with 65 real observations (nominally "enough" for a 5- + parameter fit) but all of them from a single ~2-month window + produced a wildly unreliable fit when evaluated on the rest of the + year (RMSE off by orders of magnitude), because harmonic basis + functions extrapolate poorly outside the day-of-year range they + were actually constrained by. A single busy season doesn't tell you + the shape of the whole year. + + Returns: + {column_name: coefficient_array} -- a column without enough + points OR without enough seasonal coverage is skipped entirely, + not silently given untrustworthy coefficients. + """ + doy_frac = _day_of_year_fraction(wide_df.index) + basis = _harmonic_basis(doy_frac, n_harmonics) + n_params = basis.shape[1] + months = wide_df.index.month + + coeffs = {} + for col in wide_df.columns: + y = wide_df[col].values + valid = ~np.isnan(y) + if valid.sum() < n_params: + continue + n_months_covered = len(set(months[valid])) + if n_months_covered < min_months_covered: + continue + beta, _, _, _ = np.linalg.lstsq(basis[valid], y[valid], rcond=None) + coeffs[col] = beta + return coeffs + + +def predict_climatology(coeffs: Dict[str, np.ndarray], dates: pd.DatetimeIndex, n_harmonics: int = 2) -> pd.DataFrame: + """ + Evaluates the fitted harmonic climatology at any dates -- including + dates far beyond the training data's range, which is the actual + point: climatology needs only day-of-year, not historical + proximity, so it's a legitimate forecast at day 180 the same way + it's a legitimate forecast at day 1. + """ + doy_frac = _day_of_year_fraction(dates) + basis = _harmonic_basis(doy_frac, n_harmonics) + result = {col: basis @ beta for col, beta in coeffs.items()} + return pd.DataFrame(result, index=dates) + + +def compute_anomalies(wide_df: pd.DataFrame, coeffs: Dict[str, np.ndarray], n_harmonics: int = 2) -> pd.DataFrame: + """actual - climatology, per column -- what the anomaly/deep model + would actually be trained to predict, not raw values.""" + climatology = predict_climatology(coeffs, wide_df.index, n_harmonics) + common_cols = [c for c in wide_df.columns if c in climatology.columns] + return wide_df[common_cols] - climatology[common_cols] + + +def aggregate_anomalies(daily_anomalies: pd.DataFrame, freq: str, min_days_required: float = 0.8) -> pd.DataFrame: + """ + Resamples a daily [date x column] anomaly table to a coarser + resolution ('W' for weekly, 'M' for monthly) by averaging. + + Built specifically because daily resolution is the wrong target for + long horizons -- whether a whole month runs wetter or drier than + average is a more tractable question than predicting any single + day within it, the same reason real seasonal forecast products + report monthly/seasonal aggregates rather than day-by-day forecasts + past ~2 weeks. Climatology itself stays fit at daily resolution + (already validated, no reason to refit it) -- this aggregates the + ANOMALY, which is the actual quantity a coarser-resolution model + would predict. + + A period (week/month) is only kept if at least `min_days_required` + fraction of its days have real (non-NaN) data -- a period built + from mostly-missing days would give a falsely confident average, + same principle as this project's other completeness checks. + """ + period_size = {"W": 7, "ME": 30}.get(freq) + if period_size is None: + raise ValueError(f"freq must be 'W' or 'ME', got {freq!r}") + + counts = daily_anomalies.resample(freq).count() + means = daily_anomalies.resample(freq).mean() + # resample('W')/('M') bin sizes vary slightly (a week spanning a + # month boundary, a 28-31 day month) -- use the ACTUAL day count in + # each bin as the denominator, not a fixed nominal period_size. + bin_day_counts = daily_anomalies.resample(freq).size() if hasattr(daily_anomalies.resample(freq), "size") else None + actual_period_lengths = daily_anomalies.resample(freq).apply(lambda x: len(x)) + coverage = counts / actual_period_lengths.replace(0, np.nan) + + return means.where(coverage >= min_days_required) + + +# Horizon-dependent resolution regime: which resolution and lookback to +# use for a given forecast horizon band, and the SAME horizons expressed +# in that resolution's own native units (used directly by +# prepare_training_windows, which doesn't know or care what a "1" means +# physically). Chosen because daily resolution is structurally the wrong +# target past ~2 weeks -- see aggregate_anomalies's docstring -- and to +# match C3S's own native monthly resolution at the long end, so blending +# our model with the real seasonal forecast needs no resolution +# reconciliation. +REGIME_CONFIG = { + "daily": {"freq": None, "lookback": 60, "horizons_native": [1, 5, 10, 14]}, + "weekly": {"freq": "W", "lookback": 12, "horizons_native": [3, 6, 9, 13]}, # ~15-90 days as weeks + "monthly": {"freq": "ME", "lookback": 12, "horizons_native": [3, 6]}, # ~90-180 days as months +} + + +def evaluate_climatology_skill( + wide_df: pd.DataFrame, coeffs: Dict[str, np.ndarray], train_means: Optional[Dict[str, float]] = None, + n_harmonics: int = 2, +) -> pd.DataFrame: + """ + Per-column RMSE of the climatology baseline against real observed + values -- the number the anomaly/deep model needs to beat to + justify its own complexity. + + Also reports RMSE of a flat-mean baseline, using `train_means` + (each column's mean computed from TRAINING data only) rather than + the test period's own mean. Using the test set's own mean is a + look-ahead comparison -- no real forecast can know a future + period's true average in advance -- and confirmed to matter in + practice: on one real station, climatology looked like it "lost" to + a flat-mean baseline computed this illegitimate way, when the real, + fair comparison (flat mean from training data only) tells a + different story. If train_means isn't given, falls back to the + test set's own mean with a clear label, so the illegitimate version + is never silently presented as the real comparison. + """ + climatology = predict_climatology(coeffs, wide_df.index, n_harmonics) + rows = [] + for col in wide_df.columns: + if col not in climatology.columns: + continue + actual = wide_df[col].values + valid = ~np.isnan(actual) + if valid.sum() == 0: + continue + clim_pred = climatology[col].values + clim_rmse = np.sqrt(np.mean((actual[valid] - clim_pred[valid]) ** 2)) + + if train_means is not None and col in train_means: + flat_value = train_means[col] + flat_label = "flat_train_mean_rmse" + else: + flat_value = np.nanmean(actual) + flat_label = "flat_TEST_mean_rmse (look-ahead, not a fair baseline)" + flat_rmse = np.sqrt(np.mean((actual[valid] - flat_value) ** 2)) + + rows.append({"column": col, "n_obs": int(valid.sum()), + "climatology_rmse": clim_rmse, flat_label: flat_rmse, + "climatology_beats_flat_mean": clim_rmse < flat_rmse}) + return pd.DataFrame(rows) \ No newline at end of file diff --git a/src/graph/dynamic_features.py b/src/graph/dynamic_features.py index 4e9df288d5a1a69be23cabc50dd29038b7c0f356..923c5f0a1a5a8d15214e3feb0cfe2e670ec5c18a 100644 --- a/src/graph/dynamic_features.py +++ b/src/graph/dynamic_features.py @@ -72,6 +72,343 @@ def build_discharge_timeseries( return wide +def map_stations_to_precipitation_gauges(nodes_df: pd.DataFrame, precip_stations: pd.DataFrame) -> pd.Series: + """ + Which real Météo-France precipitation station each graph node reads + from -- nearest by real haversine distance, same reasoning and same + underlying helper as map_stations_to_grid_cells (the ERA5 grid-cell + assignment): computed once, since the spatial relationship doesn't + change over time, and reused for every date. + + Unlike the ERA5 grid (a uniform ~20-30 cell mesh with predictable + spacing), real precipitation gauges are irregularly placed -- + genuinely possible for the nearest one to be tens of km away in a + sparsely-instrumented area. No distance cap is applied here: an + imperfect nearest real gauge is still real, directly-measured data, + which was the whole reason to prefer this over ERA5 reanalysis in + the first place -- capping it back down to "use reanalysis instead + beyond some radius" would undo that. + + Args: + nodes_df: real graph nodes with latitude/longitude columns. + precip_stations: real Météo-France stations with NUM_POSTE, + LAT, LON columns (as returned by + fetch_meteofrance_precipitation.py's combined output). + + Returns: + Series indexed like nodes_df, values are real NUM_POSTE station + codes. + """ + unique_stations = precip_stations.drop_duplicates(subset="NUM_POSTE") + station_lat = unique_stations["LAT"].values + station_lon = unique_stations["LON"].values + station_ids = unique_stations["NUM_POSTE"].values + + assignments = [] + for _, row in nodes_df.iterrows(): + dist = _haversine_km_vec(row["latitude"], row["longitude"], station_lat, station_lon) + assignments.append(station_ids[dist.argmin()]) + return pd.Series(assignments, index=nodes_df.index, name="precip_station_id") + + +def compute_rolling_precip_sum(precip_tensor: np.ndarray, window_days: int = 30, min_coverage: float = 0.5) -> np.ndarray: + """ + Real rolling sum of precipitation over the trailing window_days, + per node per date -- the "recent rainfall" side of the specific- + discharge relationship below. A window built from mostly-missing + days is treated as unreliable (NaN), not silently understated, + same completeness discipline already used for the per-example + lookback-window precip sum in train_spatiotemporal_gnn.py and for + aggregate_anomalies in the climate work. + """ + n_nodes, n_dates = precip_tensor.shape + rolling_sum = np.full((n_nodes, n_dates), np.nan, dtype=np.float32) + for t in range(window_days, n_dates): + window = precip_tensor[:, t - window_days:t] + real_fraction = (~np.isnan(window)).mean(axis=1) + window_sum = np.nansum(window, axis=1) + rolling_sum[:, t] = np.where(real_fraction >= min_coverage, window_sum, np.nan) + return rolling_sum + + +def fit_specific_discharge_coefficient( + discharge_tensor: np.ndarray, rolling_precip_sum: np.ndarray, catchment_area_km2: np.ndarray, + dates, train_end: str, +) -> float: + """ + Fits k in Q ~= k * (recent precipitation sum) * (catchment area) -- + a standard, well-established hydrological technique for estimating + discharge at ungauged points ("regionalization" via specific + discharge scaling), not a novel or speculative one. Fit ONLY on + real (Q, precip, area) triples at GAUGED stations during the real + TRAINING period specifically -- same "fit on train only" discipline + as standardization, avoiding any leakage from val/test into a + coefficient that then gets applied everywhere, including at + ungauged nodes evaluated during val/test. + + Closed-form least squares through the origin (no intercept -- zero + rainfall over the window should imply ~zero attributable runoff, + a physically reasonable constraint, not an arbitrary modeling + choice): k = sum(x*y) / sum(x^2), x = precip_sum * area, y = Q. + + Returns: + The fitted real scalar k. Raises if there's no real data to + fit against, rather than returning a meaningless default. + """ + train_col_mask = pd.DatetimeIndex(dates) <= pd.Timestamp(train_end) + Q_train = discharge_tensor[:, train_col_mask] + P_train = rolling_precip_sum[:, train_col_mask] + A = catchment_area_km2[:, None] * np.ones_like(Q_train) # broadcast area across all real dates + + x = P_train * A + y = Q_train + valid = ~np.isnan(x) & ~np.isnan(y) & (x != 0) + n_real = int(valid.sum()) + if n_real == 0: + raise ValueError("No real (discharge, precipitation, catchment area) triples available to fit " + "the specific-discharge coefficient -- check that all three are real for at " + "least some gauged station during the training period.") + + x_valid, y_valid = x[valid], y[valid] + k = float(np.sum(x_valid * y_valid) / np.sum(x_valid ** 2)) + print(f"Fitted specific-discharge coefficient k={k:.6g} on {n_real} real (Q, precip, area) " + f"triple(s) from gauged stations, training period only") + return k + + +def estimate_discharge_from_precip( + rolling_precip_sum: np.ndarray, catchment_area_km2: np.ndarray, k: float, +) -> np.ndarray: + """ + Applies the fitted specific-discharge coefficient to EVERY node + (gauged or not), wherever real precipitation and catchment area + both exist -- this is the actual point: an ungauged node with real + precipitation and real catchment area gets a real, physically- + motivated discharge ESTIMATE, computed the same way for every node + rather than only where discharge happens to already be known. + + This is deliberately an INPUT feature, never a target -- it must + never be treated as if it were a real observation. It's fit from + real data and grounded in a real, standard hydrological + relationship, but it is still an estimate, not a measurement, and + should reach the model exactly the way every other real-but- + imperfect input does: as a value the model can learn to weigh, + never as something the supervised loss is held accountable to + match exactly. + """ + n_nodes = catchment_area_km2.shape[0] + area_broadcast = catchment_area_km2[:, None] * np.ones((n_nodes, rolling_precip_sum.shape[1])) + estimated = k * rolling_precip_sum * area_broadcast + # NaN propagates automatically wherever rolling_precip_sum or + # catchment_area_km2 is real NaN -- no explicit masking needed here, + # multiplication by NaN is already NaN. + return estimated.astype(np.float32) + + +FORECAST_LEAD_TIMES = [1, 2, 3, 5, 7] # real confirmed coverage in previous_runs_forecast.csv -- NOT every horizon this project predicts at + + +def build_forecast_tensor( + nodes_df: pd.DataFrame, forecast_csv_path: Path, anchor_dates, horizons: List[int], +) -> Tuple[np.ndarray, np.ndarray]: + """ + Real forecasted precipitation, per (anchor_date, node, horizon) -- + built from previous_runs_forecast.csv, whose real confirmed schema + is genuinely different from every other timeseries builder in this + file: it's not a simple [date x station] wide table, it's one row + per real OBSERVATION date with several "precipitation_previous_ + dayN" columns, each holding what was FORECASTED N days before that + date, for that date. + + The real mapping this requires, worked through explicitly: for an + anchor date t and horizon h, the forecast we want is "what was + predicted h days in advance, for date t+h" -- which was ISSUED on + date (t+h) - h = t, exactly our anchor. In this file's real column + convention, that value lives in the ROW for date=t+h, in column + precipitation_previous_day{h}. Getting this backwards (e.g. using + the row for date=t instead) would silently use a forecast for the + wrong target date entirely. + + station_code in this real file already matches this project's real + graph node station codes directly (confirmed against real sample + data) -- no nearest-neighbor mapping needed here, unlike the + Meteo-France precipitation gauges, which come from a different + network entirely. + + Real, confirmed constraints, not assumptions: only horizons in + FORECAST_LEAD_TIMES ([1,2,3,5,7]) have any real forecast data at + all -- every other horizon this project predicts at (4, 10, 30, 60, + 90, 180) has NONE, and gets missing=True unconditionally, not + silently filled with anything. Real coverage also only starts + 2024-03-01 -- anchor dates before that (the large majority of this + project's 2013+ training period) get missing=True for every + horizon, real lead time or not. + + Returns: + forecast_tensor, missing_tensor: both [n_examples, n_nodes, + n_horizons], real values (or 0.0 placeholder) and an explicit + real/missing flag respectively -- same value+missingness + pattern used for every other real-but-incomplete input in this + project. + """ + df = pd.read_csv(forecast_csv_path) + df["date"] = pd.to_datetime(df["date"]) + + # Pre-indexed for fast repeated lookup -- {(date, station_code): row} + # rather than re-filtering the real dataframe for every single + # (anchor_date, node, horizon) combination, which would be + # prohibitively slow at this project's real scale (thousands of + # examples x 100 nodes x 10 horizons). + df_indexed = df.set_index(["date", "station_code"]) + + station_codes = nodes_df["station_code"].tolist() + n_nodes = len(station_codes) + n_examples = len(anchor_dates) + n_horizons = len(horizons) + + forecast_tensor = np.zeros((n_examples, n_nodes, n_horizons), dtype=np.float32) + missing_tensor = np.ones((n_examples, n_nodes, n_horizons), dtype=np.float32) # starts fully missing + + for h_idx, h in enumerate(horizons): + if h not in FORECAST_LEAD_TIMES: + continue # no real forecast data for this horizon at all -- stays fully missing + col = f"precipitation_previous_day{h}" + for ex_idx, anchor in enumerate(anchor_dates): + target_date = pd.Timestamp(anchor) + pd.Timedelta(days=h) + for node_idx, station_code in enumerate(station_codes): + try: + value = df_indexed.loc[(target_date, station_code), col] + except KeyError: + continue # genuinely no real row for this date/station -- stays missing + if pd.notna(value): + forecast_tensor[ex_idx, node_idx, h_idx] = value + missing_tensor[ex_idx, node_idx, h_idx] = 0.0 + + return forecast_tensor, missing_tensor + + +def compute_days_since_last_real(tensor, sentinel_days=365.0): + """ + For each [node, date] position, how many real days have passed + since the most recent real (non-NaN) observation at or before that + date -- 0 if the current day itself is real. Built specifically + because a binary missing-flag alone can't distinguish a single-day + sensor glitch from a station offline for months; both look + identical to the model at any given missing timestep without this. + + Computed over the FULL real date range, not per-window -- correctly + looks backward past wherever a later 30-day lookback window happens + to start, rather than resetting to "unknown" the moment a window + boundary is crossed. This is exactly why it has to be built once + here, before windowing, and then sliced the same way value/missing + already are, rather than computed fresh inside each window. + + Nodes with NO real observation at all up through a given date get + the real sentinel value (a fixed ceiling, not literal infinity) -- + "genuinely never observed" is a real, distinct case from "observed + 365+ days ago", but both should read as "don't trust this" to the + model without a numerically unbounded value reaching it. + """ + import numpy as np + n_nodes, n_dates = tensor.shape + is_real = ~np.isnan(tensor) + date_indices = np.arange(n_dates, dtype=np.float64) + + idx_where_real = np.where(is_real, date_indices[None, :], -1.0) + last_real_idx = np.maximum.accumulate(idx_where_real, axis=1) + + dt = np.where(last_real_idx < 0, sentinel_days, date_indices[None, :] - last_real_idx) + dt = np.minimum(dt, sentinel_days) + return dt.astype(np.float32) + + +def compress_days_since_last_real(dt): + """ + log(1 + dt) -- compresses the real scale so a 3-vs-10-day + distinction isn't drowned out by a 3-vs-3000-day one the way a raw + linear day count would. Same unstandardized-large-number + instability this project already hit once with raw discharge in + L/s, applied preemptively here rather than found the hard way again. + """ + import numpy as np + return np.log1p(dt).astype(np.float32) + + +def build_precipitation_timeseries( + nodes_df: pd.DataFrame, precip_csv_path: Path, date_range: Tuple[str, str], freq: str = "D", +) -> pd.DataFrame: + """ + Real daily precipitation (mm), pivoted to [date x station_code] wide + form -- same output convention as build_discharge_timeseries and + build_waterlevel_timeseries, so it drops into the same downstream + pipeline (assemble_dynamic_tensor etc.) without special-casing. + + Real Météo-France gauge data (fetch_meteofrance_precipitation.py), + not ERA5 reanalysis -- deliberately: precipitation varies sharply + over short distances in ways a reanalysis grid cell can miss, so a + real nearby gauge is more locally authoritative where one exists. + Each of OUR graph nodes reads from its own nearest real gauge (see + map_stations_to_precipitation_gauges), which can mean several of + our nodes share the same real gauge if they're close together -- + expected and correct, not a bug, the same way several ERA5-grid + climate stations already share one grid cell elsewhere in this + project. + """ + precip_df = pd.read_csv(precip_csv_path) + precip_df["date"] = pd.to_datetime(precip_df["date"]) + + assignments = map_stations_to_precipitation_gauges(nodes_df, precip_df[["NUM_POSTE", "LAT", "LON"]]) + + start, end = date_range + full_index = pd.date_range(start, end, freq=freq) + wide = pd.DataFrame(index=full_index) + wide.index.name = "date" + + for node_idx, station_id in assignments.items(): + station_code = nodes_df.loc[node_idx, "station_code"] + station_series = precip_df[precip_df["NUM_POSTE"] == station_id].set_index("date")["precip_mm_daily"] + wide[station_code] = station_series.reindex(full_index) + + return wide + + +def build_waterlevel_timeseries( + nodes_df: pd.DataFrame, hydrometric_path: Path, date_range: Tuple[str, str], + freq: str = "D", +) -> pd.DataFrame: + """ + Real daily water level (HIXnJ), pivoted to [date x station_code] + wide form -- mirrors build_discharge_timeseries exactly, using + waterlevel_mm instead of discharge_m3s (both already present in + HydrometricLoader's real output, confirmed directly earlier this + project: columns ['date', 'station_code', 'discharge_m3s', + 'waterlevel_mm']). + + Confirmed real coverage: 13 of 27 real stations have water-level + data at all -- a genuinely different, larger set than discharge's + 6-8 in-window stations, though not identical (some stations have + one variable but not the other). Built specifically to give the + model's water-level output channel real supervision for the first + time -- previously it only had one loose consistency constraint + (manning_consistency_loss) and no direct target at all. + """ + from ..data.loaders.hydrometric import HydrometricLoader + + loader = HydrometricLoader(data_path=hydrometric_path) + df = loader.load() + if "waterlevel_mm" not in df.columns: + raise ValueError("Loaded hydrometric data has no waterlevel_mm column") + + start, end = date_range + df = df[(df["date"] >= start) & (df["date"] <= end)] + wide = df.pivot_table(index="date", columns="station_code", values="waterlevel_mm", aggfunc="mean") + + full_index = pd.date_range(start, end, freq=freq) + wide = wide.reindex(full_index) + wide.index.name = "date" + return wide + + def build_groundwater_timeseries( nodes_df: pd.DataFrame, ades_path: Path, date_range: Tuple[str, str], max_distance_km: float = 20.0, freq: str = "D", @@ -155,6 +492,96 @@ def build_groundwater_timeseries( return level_wide, depth_wide +def build_climate_grid_timeseries( + safran_path: Path, bbox: Tuple[float, float, float, float], date_range: Tuple[str, str], +) -> Tuple[Dict[str, pd.DataFrame], pd.DataFrame]: + """ + Real ERA5 climate at native 0.25 deg grid resolution -- not + interpolated per station. Confirmed against real data that + station-level interpolation was redundant: several real stations + landed on bit-identical values for the same variable, since they + share the same underlying grid cell. This is the actual fix, not + a bigger version of the same approach. + + Returns: + (climate_dict, grid_coords) -- + climate_dict: {variable_name: wide DataFrame [date x grid_cell_id]}, + same shape/contract as build_climate_timeseries's return, so + every downstream consumer (climatology, later the anomaly + model) works completely unchanged against either. + grid_coords: DataFrame [grid_cell_id, grid_lat, grid_lon] -- + needed by map_stations_to_grid_cells (and later, by any river + node) to know which real coordinate each cell ID refers to. + """ + from ..data.loaders.safran import SAFRANLoader + + loader = SAFRANLoader(data_path=safran_path) + df = loader.load_grid(bbox=bbox) + df = loader.convert_units(df) + + start, end = date_range + df = df[(df["date"] >= start) & (df["date"] <= end)] + + df["grid_cell_id"] = df["grid_lat"].round(3).astype(str) + "_" + df["grid_lon"].round(3).astype(str) + grid_coords = df.drop_duplicates("grid_cell_id")[["grid_cell_id", "grid_lat", "grid_lon"]].reset_index(drop=True) + + # convert_units() ADDS converted columns rather than replacing the raw + # ones -- confirmed against real data that both temp_2m_K and temp_C + # (and every other raw/converted pair) were being fit separately, + # doubling work for identical underlying signal (a linear unit shift + # produces the exact same climatology skill profile either way). + # wind_u_ms/wind_v_ms are NOT dropped alongside wind_speed_ms -- unlike + # the other pairs, speed is a genuinely different derived quantity + # (loses direction information u/v carry), not a unit conversion of + # either component. + RAW_COLUMNS_SUPERSEDED_BY_CONVERTED = { + "temp_2m_K", "precip_m", "evap_m", "solar_Jm2", "snow_m", "runoff_m", + } + var_cols = [c for c in df.columns + if c not in ("date", "grid_lat", "grid_lon", "grid_cell_id") + and c not in RAW_COLUMNS_SUPERSEDED_BY_CONVERTED] + climate_dict = {} + for var in var_cols: + wide = df.pivot_table(index="date", columns="grid_cell_id", values=var, aggfunc="mean") + climate_dict[var] = wide + return climate_dict, grid_coords + + +def map_stations_to_grid_cells(nodes_df: pd.DataFrame, grid_coords: pd.DataFrame) -> pd.Series: + """ + Which grid cell each real node reads its climate forecast from -- + computed once (the spatial join is time-invariant, same principle + already used for groundwater's nearby-well lookup), not repeated + per timestep or per query. Nearest-cell by exact haversine -- at + only ~20-30 candidate cells for this study area, no KD-tree + prefilter is needed the way it was for ADES's ~100 wells against + ~4,500 nodes; a direct distance computation per node against every + cell is cheap at this scale. + + Returns: + Series indexed like nodes_df, values are grid_cell_id strings + matching build_climate_grid_timeseries's wide DataFrame columns. + """ + cell_lat = grid_coords["grid_lat"].values + cell_lon = grid_coords["grid_lon"].values + cell_ids = grid_coords["grid_cell_id"].values + + assignments = [] + for _, row in nodes_df.iterrows(): + dist = _haversine_km_vec(row["latitude"], row["longitude"], cell_lat, cell_lon) + assignments.append(cell_ids[dist.argmin()]) + return pd.Series(assignments, index=nodes_df.index, name="grid_cell_id") + + +def _haversine_km_vec(lat1, lon1, lat2, lon2): + R = 6371.0 + lat1r, lon1r = np.radians(lat1), np.radians(lon1) + lat2r, lon2r = np.radians(lat2), np.radians(lon2) + dlat, dlon = lat2r - lat1r, lon2r - lon1r + a = np.sin(dlat / 2) ** 2 + np.cos(lat1r) * np.cos(lat2r) * np.sin(dlon / 2) ** 2 + return R * 2 * np.arcsin(np.sqrt(a)) + + def build_climate_timeseries( nodes_df: pd.DataFrame, safran_path: Path, date_range: Tuple[str, str], ) -> Dict[str, pd.DataFrame]: @@ -163,11 +590,14 @@ def build_climate_timeseries( station_code] wide form -- skips node_features.py's add_safran_ features aggregation step entirely rather than reversing it. - UNTESTED in this environment: no real era5_*.nc files were - available here (only ADES and hydrometric real data were). The - logic mirrors build_discharge_timeseries's pivot pattern directly, - but please verify the real output shape/values before trusting it - for training. + SUPERSEDED for new work by build_climate_grid_timeseries + + map_stations_to_grid_cells: confirmed against real ERA5 data (the + real climatology test run) that several stations land on + bit-identical values here, since ERA5's 0.25 deg cells are coarser + than the spacing between some real gauges -- this function still + works and is now real-data-tested, but it's doing per-station + interpolation onto what's provably a shared, coarser grid. Kept for + any caller that specifically wants a station-indexed table. """ from ..data.loaders.safran import SAFRANLoader @@ -181,7 +611,11 @@ def build_climate_timeseries( start, end = date_range df = df[(df["date"] >= start) & (df["date"] <= end)] - var_cols = [c for c in df.columns if c not in ("date", "station_code")] + RAW_COLUMNS_SUPERSEDED_BY_CONVERTED = { + "temp_2m_K", "precip_m", "evap_m", "solar_Jm2", "snow_m", "runoff_m", + } + var_cols = [c for c in df.columns + if c not in ("date", "station_code") and c not in RAW_COLUMNS_SUPERSEDED_BY_CONVERTED] result = {} for var in var_cols: wide = df.pivot_table(index="date", columns="station_code", values=var, aggfunc="mean") diff --git a/src/graph/node_features.py b/src/graph/node_features.py index 0dd06d8094774fb7b093884e047b037c84e14ba1..7c2e184ec79ea35633a9d258b6da33232b9788fe 100644 --- a/src/graph/node_features.py +++ b/src/graph/node_features.py @@ -142,6 +142,18 @@ def add_geology_features(nodes_df: pd.DataFrame, bdcharm_path: Path) -> pd.DataF not an ordered quantity, so a raw code would let build_pyg_graph's numeric auto-detection z-score it as if formation codes had a meaningful numeric ordering, which they don't. + + Needs geopandas. Searches defensively for the geological-formations + shapefile (filename containing "FGEOL", BD Charm-50's documented + naming for that layer) and a plausible attribute column, rather than + assuming an exact filename/column name -- the real schema for these + files hasn't been directly inspected, so guessing wrong silently + would be worse than searching and reporting what was found. + + Not independently verified against a real BD Charm-50 shapefile -- + check the printed shapefile/column choice against what you actually + see when you open one of these files yourself before trusting the + result. """ import geopandas as gpd from shapely.geometry import Point @@ -194,22 +206,49 @@ def add_geology_features(nodes_df: pd.DataFrame, bdcharm_path: Path) -> pd.DataF def add_cavites_features( - nodes_df: pd.DataFrame, bdcavites_path: Path, max_distance_km: float = 20.0 + nodes_df: pd.DataFrame, bdcavites_path: Path, max_distance_km: float = 20.0, + allowed_types: Optional[List[str]] = None, ) -> pd.DataFrame: """ Distance to nearest known cavity/sinkhole + count within - max_distance_km, from scripts/download_bdcavites.py's + max_distance_km, from scripts/data_acquisition/download_bdcavites.py's cavite_localisee.geojson (BRGM's national cavity inventory, via Géorisques' WFS). + allowed_types filters on the real "type_cavite" field -- CONFIRMED + NECESSARY, not optional, against this project's actual real data: + an unfiltered check across all 27 real gauges found 24/27 within + 1km of *some* cavity, with counts in the thousands, because + BDCavités catalogs every cavity type and the real breakdown is + dominated by quarries (carrière, 21,813 of the national file), + civil/military structures (~9,900), and cellars (cave, 3,990) -- + none hydrologically meaningful. Only "naturelle" (8,313) are real + natural cavities, the category that could actually indicate genuine + karst/water-loss connectivity -- confirmed as the right default via + scripts/analysis/check_cavity_proximity.py's real, direct + comparison against the known bétoire stations. "indéterminé" is + genuinely ambiguous (could be real unclassified natural features or + unclassified quarries) and is NOT included by default -- pass it + explicitly if a broader, less conservative pass is wanted. + Same KD-tree coarse-prefilter + exact-haversine pattern already validated for add_groundwater_features -- point data at unknown real density, so avoid assuming either a fast-enough naive loop or guessing at scale ahead of time. + + Graceful on an empty/missing file: BDCavités coverage is genuinely + incomplete in places (built department-by-department, not all + departments finished on the same timeline -- see the department- + coverage caveat in download_bdcavites.py's docstring), so zero + cavities found near a station is plausible real data, not + necessarily evidence the fetch failed. """ import json from scipy.spatial import cKDTree + if allowed_types is None: + allowed_types = ["naturelle"] + path = Path(bdcavites_path) if not path.exists(): raise FileNotFoundError(f"{path} not found") @@ -225,14 +264,27 @@ def add_cavites_features( return out cavity_coords = [] + n_filtered = 0 for f in features: geom = f.get("geometry", {}) coords = geom.get("coordinates") - if geom.get("type") == "Point" and coords: - cavity_coords.append((coords[1], coords[0])) # (lat, lon) + if geom.get("type") != "Point" or not coords: + continue + real_type = f.get("properties", {}).get("type_cavite") + if real_type not in allowed_types: + n_filtered += 1 + continue + cavity_coords.append((coords[1], coords[0])) # (lat, lon) cavity_lat = np.array([c[0] for c in cavity_coords]) cavity_lon = np.array([c[1] for c in cavity_coords]) - print(f"add_cavites_features: {len(cavity_coords)} point cavity feature(s) loaded from {path}") + print(f"add_cavites_features: {len(cavity_coords)} real cavity point(s) matching " + f"type_cavite in {allowed_types} loaded from {path} ({n_filtered} filtered out by type)") + + if len(cavity_coords) == 0: + out = nodes_df.copy() + out["distance_to_nearest_cavity_km"] = float("nan") + out[f"n_cavities_within_{int(max_distance_km)}km"] = 0 + return out tree = cKDTree(np.column_stack([cavity_lon, cavity_lat])) coarse_radius_deg = (max_distance_km / 111.0) * 1.5 # see add_groundwater_features for rationale @@ -264,16 +316,16 @@ def add_ndvi_features(nodes_df: pd.DataFrame, ndvi_path: Path) -> pd.DataFrame: ESA WorldCover Sentinel-2 NDVI yearly percentile composite (p10/p50/p90) per station, from scripts/fetch_worldcover_ndvi.py. - NOTE: raw values are VITO's scaled digital numbers, not independently - confirmed to be literal -1..1 NDVI (no access to their exact scale/ - offset convention in this environment) -- internally consistent - (p90 > p50 > p10 holds for every real station checked), so the - relative signal is trustworthy even if the absolute units aren't - pinned down. Doesn't block use as a model feature: build_pyg_graph - z-scores every feature anyway, which is invariant to an unknown - linear scaling. Only matters if literal NDVI units are needed later - (e.g. for plotting against a textbook NDVI range) -- resolve the - scale/offset from VITO's product documentation before then. + NOTE: raw values are VITO's scaled digital numbers, not confirmed to + be literal -1..1 NDVI (their exact scale/offset convention hasn't + been resolved) -- internally consistent (p90 > p50 > p10 holds for + every real station checked), so the relative signal is trustworthy + even if the absolute units aren't pinned down. Doesn't block use as + a model feature: build_pyg_graph z-scores every feature anyway, + which is invariant to an unknown linear scaling. Only matters if + literal NDVI units are needed later (e.g. for plotting against a + textbook NDVI range) -- resolve the scale/offset from VITO's product + documentation before then. Same coverage limitation as add_catchment_features/add_landcover_ features: exact station_code match, real gauges only for now. diff --git a/src/graph/physics_losses.py b/src/graph/physics_losses.py index fa7f78e34cf587d42ac6d0859b0cfd61c126e6ad..ac49807c6627b637ad02a6cda2a4b9d1f0247a4d 100644 --- a/src/graph/physics_losses.py +++ b/src/graph/physics_losses.py @@ -262,12 +262,132 @@ def routing_consistency_loss(Q, routing_index: List[Tuple[int, int, int]]): return _mse(residual_cat) +def _soft_dtw_pair_numpy(x: np.ndarray, y: np.ndarray, gamma: float) -> float: + """Reference implementation for the numpy diagnostic path and for + validating the torch path below.""" + n, m = len(x), len(y) + D = (x[:, None] - y[None, :]) ** 2 + NEG_INF = 1e8 + R = np.zeros((n + 1, m + 1)) + R[0, 1:] = NEG_INF + R[1:, 0] = NEG_INF + for i in range(1, n + 1): + for j in range(1, m + 1): + vals = np.array([-R[i - 1, j] / gamma, -R[i - 1, j - 1] / gamma, -R[i, j - 1] / gamma]) + max_val = vals.max() + softmin = -gamma * (max_val + np.log(np.sum(np.exp(vals - max_val)))) + R[i, j] = D[i - 1, j - 1] + softmin + return float(R[n, m]) + + +def _soft_dtw_pair_torch(x: "torch.Tensor", y: "torch.Tensor", gamma: float) -> "torch.Tensor": + """ + Differentiable soft-DTW between two 1-D sequences (Cuturi & Blondel, + 2017). Builds the DP table as a list-of-lists of fresh scalar + tensors rather than in-place indexed writes into a preallocated + tensor, since in-place writes into a tensor tracked by autograd can + silently break or corrupt gradients. A pure-Python double loop is + fine here given these sequences are short (ROUTING_HORIZON_COUNT + days); a vectorized anti-diagonal version isn't needed at this + length. + """ + n, m = x.shape[0], y.shape[0] + NEG_INF = torch.tensor(1e8, dtype=x.dtype, device=x.device) + ZERO = torch.zeros((), dtype=x.dtype, device=x.device) + + D = (x.unsqueeze(1) - y.unsqueeze(0)) ** 2 # [n, m] + + R = [[None] * (m + 1) for _ in range(n + 1)] + R[0][0] = ZERO + for j in range(1, m + 1): + R[0][j] = NEG_INF + for i in range(1, n + 1): + R[i][0] = NEG_INF + + for i in range(1, n + 1): + for j in range(1, m + 1): + r0, r1, r2 = R[i - 1][j], R[i - 1][j - 1], R[i][j - 1] + stacked = torch.stack([-r0 / gamma, -r1 / gamma, -r2 / gamma]) + softmin = -gamma * torch.logsumexp(stacked, dim=0) + R[i][j] = D[i - 1, j - 1] + softmin + + return R[n][m] + + +def routing_soft_dtw_loss(Q, routing_index: List[Tuple[int, int, int]], gamma: float = 0.001): + """ + Complements routing_consistency_loss rather than replacing it: that + loss assumes a fixed, distance/slope-derived lag is exactly right; + this one compares the predicted discharge shape at an upstream and + downstream node over the same contiguous window via soft-DTW's + elastic time-alignment, tolerant of the lag being slightly off. + + Keeps the fixed-lag term rather than dropping it, since letting + timing be entirely learned with no physical anchor risks the same + non-convergence Manning's roughness has shown on real edges. + + Args: + Q: predicted discharge, [n_nodes, T]. Always the model's own + dense prediction, never NaN, so no missingness masking is + needed here (unlike the sparse-ground-truth losses above). + routing_index: from build_routing_index, same (up_idx, down_idx, + lag) triples routing_consistency_loss uses. + gamma: soft-DTW smoothing -- smaller tracks literal (hard) DTW + more closely, larger gives a softer, more averaged alignment. + Also directly controls a real, confirmed failure mode: soft- + DTW is NOT guaranteed non-negative except in the gamma->0 + limit (where it converges to true, hard DTW). Two real + rounds of validation went into this default, not one -- + the first (gamma=1.0 -> 0.1) was tested against independent + random sequences and looked safe (0/200 negative), but real + routing edges connect upstream/downstream nodes on the same + river, which are highly correlated with small real + differences, not independent -- a genuinely different data + shape that gamma=0.1 turned out NOT to be safe against + (121-301/500 negative depending on scale, confirmed + directly once real training logs showed this loss reading + exactly 0 -- floored by the defensive clamp -- on every + single real example). gamma=0.001 showed 0/500 negative + against BOTH the realistic correlated case and the original + independent case, with a comfortable positive margin in + both. Validate any change to this default against a + correlated-sequence test, not just an independent one -- + that's the actual gap that caused this to need fixing twice. + + Returns: + Scalar loss (0.0 if no edges are usable), averaged across edges, + floored at 0 as a second layer of protection beyond gamma's own + effect -- soft-DTW's negative-value risk shrinks with gamma but + isn't provably eliminated at any finite value, and a loss term + that can go negative is a real problem for gradient-based + training regardless of how rarely it happens. + """ + T = Q.shape[1] + use_torch = _HAS_TORCH and isinstance(Q, torch.Tensor) + pair_fn = _soft_dtw_pair_torch if use_torch else _soft_dtw_pair_numpy + + distances = [] + for up_idx, down_idx, lag in routing_index: + if T < 2: + continue + x = Q[up_idx, :] + y = Q[down_idx, :] + distances.append(pair_fn(x, y, gamma)) + + if not distances: + return Q.sum() * 0.0 if use_torch else 0.0 + + if use_torch: + return torch.clamp(torch.stack(distances).mean(), min=0.0) + return float(max(0.0, np.mean(distances))) + + # --------------------------------------------------------------------------- # 4. Water balance: P - ET - Q - deltaS ~= 0, per node # --------------------------------------------------------------------------- def water_balance_loss( - Q_m3s, + Q_Ls, precip_mm, evap_mm, catchment_area_km2, @@ -278,12 +398,26 @@ def water_balance_loss( Precipitation minus evapotranspiration minus discharge minus storage change should balance to ~0, in volume terms, over the given period. - UNIT CONVERSION (the easy part to get subtly wrong): 1 mm of depth - over 1 km^2 is 1000 m^3 (1 km^2 = 1e6 m^2, 1 mm = 1e-3 m, - 1e6 * 1e-3 = 1e3). P and ET (mm, over the period) get converted to - m^3 via catchment_area_km2 before comparing against Q, which is - converted from a rate (m^3/s) to a volume by multiplying by the - period length in seconds. + UNIT CONVERSION (the easy part to get subtly wrong, and confirmed as + a REAL bug found while implementing Manning's equation, not just a + hypothetical risk): this function's discharge argument was + previously named/documented as Q_m3s and used directly in m^3/s + math with no conversion -- but this project's real discharge data + is confirmed to actually be in L/s despite column names like + "target_discharge_m3s" implying otherwise (checked directly: real + observed values of 7,095-26,063 for a ~500km2 catchment are + physically absurd as m^3/s -- Amazon-scale flooding -- but exactly + plausible as L/s, 7.1-26.1 m^3/s). Renamed to Q_Ls and converted + internally (/1000) specifically so callers don't have to remember + to convert before calling, which is exactly the kind of thing that + would have silently corrupted this loss by 1000x the first time it + was actually wired into training. + + 1 mm of depth over 1 km^2 is 1000 m^3 (1 km^2 = 1e6 m^2, 1 mm = + 1e-3 m, 1e6 * 1e-3 = 1e3). P and ET (mm, over the period) get + converted to m^3 via catchment_area_km2 before comparing against Q, + which is converted from a rate (L/s -> m^3/s) to a volume by + multiplying by the period length in seconds. delta_storage_m3 defaults to zero (a steady-state approximation) -- we have no direct storage measurement (soil moisture, groundwater @@ -295,7 +429,8 @@ def water_balance_loss( groundwater level trend where well coverage allows it). Args: - Q_m3s: predicted discharge, shape [n_nodes] (period-average rate). + Q_Ls: predicted discharge in L/s (this project's real native + unit), shape [n_nodes] (period-average rate). precip_mm, evap_mm: node features, already available. catchment_area_km2: from catchment.py -- NaN for ungauged/unknown catchments, in which case that node is excluded from this @@ -307,22 +442,297 @@ def water_balance_loss( Returns: Scalar loss, computed only over nodes with a real catchment area. """ - valid = ~np.isnan(catchment_area_km2) if not _HAS_TORCH or not isinstance(catchment_area_km2, torch.Tensor) \ + valid_area = ~np.isnan(catchment_area_km2) if not _HAS_TORCH or not isinstance(catchment_area_km2, torch.Tensor) \ else ~torch.isnan(catchment_area_km2) - + # precip_mm has its OWN, separate real NaN pattern (nodes with + # insufficient real precipitation coverage in the lookback window -- + # see recent_precip_sum's completeness check in the training + # script), independent of catchment_area_km2's. Both need excluding + # from `valid`, and both need safe-filling below -- fixing only one + # would leave the exact same 0*NaN=NaN gradient corruption for + # whichever one was missed. + valid_precip = ~np.isnan(precip_mm) if not _HAS_TORCH or not isinstance(precip_mm, torch.Tensor) \ + else ~torch.isnan(precip_mm) + valid = valid_area & valid_precip + + # Real, distinct bug from the smoothness fix above -- confirmed + # directly via gradient measurement AFTER that fix was already in + # place, so this is a second, separate issue, not evidence the + # first fix was wrong. Q_median (the learnable input) feeds + # Q_volume_m3 for EVERY node, including the ~84% with real NaN + # catchment_area_km2 -- so P_volume_m3, reference_volume, and + # relative_residual are all genuinely NaN at those positions too, + # even though residual[valid] correctly EXCLUDES them from the + # final loss VALUE. That exclusion does not protect the GRADIENT: + # PyTorch's chain rule still computes a LOCAL derivative at the + # excluded positions, and even though its outer contribution is + # multiplied by zero (from masking), 0 * NaN = NaN in IEEE754 + # floating point, not 0 -- confirmed directly, not assumed. That + # NaN then corrupts the gradient for the whole Q_median tensor. + # Fixed the same way missing data is handled everywhere else in + # this project: NaN is replaced with a safe, finite placeholder + # BEFORE any arithmetic touches it, and the real `valid` mask is + # applied only at the very end, for loss selection -- so no NaN + # ever enters the computational graph in the first place, at any + # position, masked or not. + safe_catchment_area_km2 = torch.where(valid, catchment_area_km2, torch.ones_like(catchment_area_km2)) \ + if _HAS_TORCH and isinstance(catchment_area_km2, torch.Tensor) \ + else np.where(valid, catchment_area_km2, 1.0) + safe_precip_mm = torch.where(valid, precip_mm, torch.zeros_like(precip_mm)) \ + if _HAS_TORCH and isinstance(precip_mm, torch.Tensor) \ + else np.where(valid, precip_mm, 0.0) + + Q_m3s = Q_Ls / 1000.0 period_seconds = period_days * 86400.0 Q_volume_m3 = Q_m3s * period_seconds - P_volume_m3 = precip_mm * catchment_area_km2 * 1000.0 - ET_volume_m3 = evap_mm * catchment_area_km2 * 1000.0 + P_volume_m3 = safe_precip_mm * safe_catchment_area_km2 * 1000.0 + ET_volume_m3 = evap_mm * safe_catchment_area_km2 * 1000.0 dS = delta_storage_m3 if delta_storage_m3 is not None else (Q_m3s * 0.0) residual = P_volume_m3 - ET_volume_m3 - Q_volume_m3 - dS - residual_valid = residual[valid] + + # Normalized to a RELATIVE (dimensionless) imbalance, not an + # absolute m^3 residual -- confirmed as a real, necessary fix, not + # a refinement: for a real, large catchment (5935 km2, confirmed + # against real data), even a genuinely period-matched residual + # reached ~1e15 when squared, purely because absolute volumes scale + # with catchment area regardless of how well the model performs. + # Dividing by a reference volume (the larger of P_volume/Q_volume, + # so a real near-zero-flow period doesn't produce a division by a + # near-zero denominator) gives a fractional imbalance instead -- + # directly comparable in scale to this project's other physics + # losses, rather than one term structurally dominating every other + # by orders of magnitude. + # Confirmed as a real, precise, isolated bug -- measured directly + # via per-term gradient checks against real training data, not + # inferred from code review: this term's FORWARD value was always + # finite and reasonable, but its GRADIENT was non-finite in every + # single real example checked (25/25), while every other physics + # term's gradient was finite in all 25. torch.maximum(torch.abs(a), + # torch.abs(b)) has real non-smooth points in its gradient -- at a + # genuine tie (a==b) or at a==0/b==0 -- and real data evidently hit + # one of those points for some of this run's 16 real catchment-area + # nodes. Replaced with a smooth L2-norm-style denominator, which has + # no non-smooth points anywhere (its gradient is well-defined for + # every real input, not just almost every input). + reference_volume = np.sqrt(P_volume_m3**2 + Q_volume_m3**2 + 1e-6) if not _HAS_TORCH or not isinstance(Q_Ls, torch.Tensor) \ + else torch.sqrt(P_volume_m3**2 + Q_volume_m3**2 + 1e-6) + relative_residual = residual / reference_volume + + residual_valid = relative_residual[valid] if (residual_valid.shape[0] if hasattr(residual_valid, "shape") else len(residual_valid)) == 0: - return Q_m3s.sum() * 0.0 + return Q_Ls.sum() * 0.0 return _mse(residual_valid) +def manning_consistency_loss( + Q_Ls, H_mm, edge_index, edge_attr, manning_n_logit, log_bank_height_m, gate_sharpness: float = 3.0, +): + """ + Manning's equation as an explicit cross-check between the model's + two outputs: Q = (1/n) * A * R^(2/3) * S^(1/2). Unlike every other + physics term here, which constrains discharge across space (a + confluence, a routing lag) or against external mass-balance + inputs, this is the first constraint tying discharge and water + level TOGETHER -- nothing previously enforced the model's two + predictions were physically consistent with each other at all. + + Manning's n (channel roughness) has no real measured value in this + project's data -- learned as a single scalar (log-parameterized so + n = exp(log_n) is always positive, the same trick already proven + for the climate model's tau), initialized near a real, physically + reasonable value for a natural stream (n around 0.03) rather than + an arbitrary starting point. + + BANK-HEIGHT RELAXATION -- the actual reason this function exists in + its current form, not just Manning's equation on its own. In-channel + Manning's equation (a single rectangular cross-section) stops being + physically true the moment water goes overbank -- flow spreads onto + a wider, rougher floodplain with completely different geometry. A + full compound-channel model would need real floodplain width and + roughness, neither of which exist in this project's data (no DEM, + no floodplain survey) -- three more ungrounded parameters on top of + an already-uncertain n. Instead: a smooth sigmoid gate multiplies + the loss weight, keyed to a single learned bank_height_m threshold + (log-parameterized for positivity, same pattern as n). Well below + the threshold, gate is ~1 (Manning's is enforced normally). Well + above it, gate decays toward 0 (the model stops being penalized for + deviating from in-channel physics exactly where that physics + genuinely no longer applies). This directly targets the real, + named gap from the physics-discussion: previously nothing in this + model's physics had anything to say about flood conditions + specifically -- the exact regime flood prediction cares about most. + + gate_sharpness controls how quickly the transition happens around + the threshold -- fixed rather than learned, deliberately: bank_ + height and n are already two learned scalars trained against a + genuinely small real dataset (100-node subgraph, sparse real + supervision); a third learned parameter controlling the SHAPE of + the relaxation risks being underconstrained rather than adding real + value. A fixed, reasonable sharpness (transition occurs over roughly + +/-1m around the threshold at the default value) is more honest + given the data available than pretending this shape is learnable + from what little real high-water data exists. + + UNITS: this project's real discharge is confirmed L/s, water level + is mm (per node_features.py's target_waterlevel_mm convention) -- + both converted internally to Manning's native SI units (m^3/s, m) + rather than assuming callers remember to convert, the same + discipline just applied to water_balance_loss above. + + Channel geometry: rectangular-channel approximation (real + trapezoidal/natural cross-sections aren't available -- see + compute_edge_width.py's own docstring on this same limitation), + using each edge's real width_m and slope (elevation_drop_m / + distance_km), and the water level PREDICTED at the edge's + downstream node as the depth proxy. + + Args: + Q_Ls: [n_nodes] predicted discharge, L/s. + H_mm: [n_nodes] predicted water level, mm. + edge_index: [2, n_edges]. + edge_attr: [n_edges, 5] -- (distance_km, elevation_drop_m, + verified_continuous, width_m, width_missing_flag), matching + combine_basins' real construction. + manning_n_logit: scalar learnable parameter (nn.Parameter in the + real model; a plain float/array here works identically for + testing without torch). + log_bank_height_m: scalar learnable parameter, same convention. + gate_sharpness: fixed (not learned -- see above), controls how + quickly the relaxation transitions around the threshold. + + Returns: + Scalar loss, computed only over edges with real (non-missing) + width data and verified_continuous=True -- the bétoire karst + stretch, where the channel doesn't behave as a normal + conveyance, is deliberately excluded, same as it's excluded + from routing. Each included edge's contribution is additionally + weighted by the bank-height gate before averaging. + """ + target = edge_index[1] + distance_km, elevation_drop_m = edge_attr[:, 0], edge_attr[:, 1] + verified_continuous, width_m, width_missing = edge_attr[:, 2], edge_attr[:, 3], edge_attr[:, 4] + + is_np = not _HAS_TORCH or not isinstance(Q_Ls, torch.Tensor) + exp_fn = np.exp if is_np else torch.exp + sqrt_fn = np.sqrt if is_np else torch.sqrt + sigmoid_fn = (lambda x: 1.0 / (1.0 + np.exp(-x))) if is_np else torch.sigmoid + clamp_fn = (lambda x, lo: np.clip(x, lo, None)) if is_np else (lambda x, lo: torch.clamp(x, min=lo)) + clamp_range_fn = (lambda x, lo, hi: np.clip(x, lo, hi)) if is_np else (lambda x, lo, hi: torch.clamp(x, min=lo, max=hi)) + + slope = elevation_drop_m / (distance_km * 1000.0 + 1e-9) + slope = clamp_fn(slope, 1e-6) # Manning's eq assumes positive slope; guard real data noise (near-flat/negative segments) + + depth_m = H_mm[target] / 1000.0 + # Real, not hypothetical, failure mode: an untrained model's very + # first H_pred values are essentially random, unstandardized + # numbers with no realistic-range guarantee -- and Manning's + # equation scales roughly with depth^(5/3), so a wild early guess + # can overflow to inf in float32 before the model has learned + # anything. That inf then poisons every later forward pass once it + # corrupts a weight via backward(). Upper-bounded at 15m -- a + # physically generous ceiling for these rivers even during a major + # flood, not a realistic operating value -- specifically to stop + # the explosion at its source, not just catch it downstream. + depth_m = clamp_range_fn(depth_m, 1e-3, 15.0) + + area = width_m * depth_m + wetted_perimeter = width_m + 2.0 * depth_m + hydraulic_radius = area / (wetted_perimeter + 1e-9) + # Same class of steep-local-derivative risk as n below, via a + # different operation: d(x^(2/3))/dx = (2/3)*x^(-1/3) also diverges + # as x->0, and depth_m's existing lower clamp (1e-3) is small enough + # that hydraulic_radius could still land in this dangerous region + # for some real (width, depth) combinations. Bounded away from zero + # for the same reason n is bounded below -- prevents this specific + # operation's gradient from being able to explode, independent of + # whatever else does or doesn't turn out to have caused a given + # real failure. + hydraulic_radius = clamp_fn(hydraulic_radius, 0.01) + + # n bounded to a physically real range for natural channels via a + # sigmoid reparameterization, not a hard clamp on an exponentiated + # value -- replaces an earlier log-parameterization + torch.clamp + # that had a confirmed, real failure mode: clamp's gradient is + # exactly zero outside its range (verified directly via finite + # differences), so once exp(the raw parameter) drifted past 0.3 or + # below 0.01, Manning's loss could never pull it back -- confirmed + # as the real cause of n landing at 0.38-1.0 across several actual + # training runs and never converging. n = 0.01 + 0.29*sigmoid(raw) + # is always in [0.01, 0.3] by construction, and its gradient, while + # it does shrink far from the center, never hits exactly zero the + # way a hard clamp's does -- so the optimizer always retains at + # least some signal to correct course. + n = 0.01 + 0.29 * sigmoid_fn(manning_n_logit) + Q_manning_m3s = (1.0 / n) * area * hydraulic_radius ** (2.0 / 3.0) * sqrt_fn(slope) + Q_manning_Ls = Q_manning_m3s * 1000.0 + + Q_model_Ls = Q_Ls[target] + residual = Q_manning_Ls - Q_model_Ls + + # Bank-height relaxation gate: ~1 well below the learned threshold + # (Manning's enforced normally), decaying toward ~0 well above it + # (the model isn't penalized for deviating from in-channel physics + # once conditions are genuinely overbank). See the module docstring + # above for why this is a fixed-sharpness sigmoid rather than a + # full compound-channel model. bank_height_m also bounded (0.1m to + # 20m) for the same reason as n above -- the sigmoid gate itself + # saturates naturally so this is lower-risk than n was, but bounding + # it costs nothing and closes the same class of failure mode. + bank_height_m = clamp_range_fn(exp_fn(log_bank_height_m), 0.1, 20.0) + gate = sigmoid_fn(-gate_sharpness * (depth_m - bank_height_m)) + + valid = (width_missing < 0.5) & (verified_continuous > 0.5) + residual_valid = residual[valid] + gate_valid = gate[valid] + n_valid = residual_valid.shape[0] if hasattr(residual_valid, "shape") else len(residual_valid) + if n_valid == 0: + return Q_Ls.sum() * 0.0 + + # Gate-weighted mean squared residual, NaN-safe (same principle as + # _mse -- a genuinely missing value must not poison every other + # edge's real contribution), but weighted rather than uniform: an + # edge that's gated toward 0 (deep overbank water) should barely + # count toward this loss, not be averaged in as if Manning's + # equation were still fully binding there. + # + # Divides by the COUNT of real edges, not the SUM of gate weights -- + # confirmed as a real bug during testing, not a hypothetical one: + # dividing by the weight sum makes this a weighted MEAN, and for a + # single edge (or edges all near the same gate value), gate*r^2 / + # gate collapses back to just r^2 regardless of how small gate is -- + # the relaxation completely cancels out. Dividing by count instead + # means an edge with gate~0 genuinely contributes ~0 to the total, + # which is the actual, intended behavior: an overbank edge's + # contribution should shrink in absolute terms, not just lose + # relative standing within an average. + # + # isfinite, not isnan, here and below -- confirmed as a real, not + # hypothetical, gap: an untrained model produced an actually- + # infinite prediction early in training, and isnan(inf) is False, + # so it sailed straight past a nan-only check into a real training + # run's validation loss (observed directly: "val supervised loss = + # inf", not nan). depth_m is now upper-bounded above, but Q_model_Ls + # itself (the model's own, equally unstandardized-random early + # discharge output) could independently cause the same failure, so + # this checks both explicitly rather than trusting the depth clamp + # alone to be the only place this could go wrong. + is_finite = np.isfinite(residual_valid) if is_np else torch.isfinite(residual_valid) + has_any = is_finite.any() if hasattr(is_finite, "any") else any(is_finite) + if not has_any: + return residual_valid.sum() * float("nan") if not is_np else np.nan + + weighted_sq = gate_valid[is_finite] * (residual_valid[is_finite] ** 2) + n_finite = is_finite.sum() if hasattr(is_finite, "sum") else sum(is_finite) + # Final defense-in-depth ceiling -- even a finite-but-astronomically- + # large residual (e.g. a partially-trained model still producing + # unrealistic values) could overflow when squared and summed; capping + # each term before summing means this function can never itself + # return inf, regardless of what produced the residual. + weighted_sq = np.minimum(weighted_sq, 1e12) if is_np else torch.clamp(weighted_sq, max=1e12) + return weighted_sq.sum() / n_finite + + # --------------------------------------------------------------------------- # Combined loss # --------------------------------------------------------------------------- @@ -364,4 +774,158 @@ def physics_informed_loss( if name in losses: total = total + weights.get(w_key, 1.0) * losses[name] losses["total"] = total - return losses \ No newline at end of file + return losses + + +# --------------------------------------------------------------------------- +# 6. NSE (Nash-Sutcliffe Efficiency): variance-tracking, a genuinely +# different failure mode from pinball loss's calibration objective +# --------------------------------------------------------------------------- + +def compute_per_node_historical_mean(target_tensor: np.ndarray) -> np.ndarray: + """ + Per-node mean of whatever real (non-NaN) observations exist in + target_tensor. Meant to be computed once from the training-period + tensor (matching how discharge_mean/discharge_std are computed + elsewhere: fit on train, never on val/test), not recomputed + per-batch -- a within-batch mean would be noisy for exactly the + nodes that need this most. + + Nodes with zero real observations anywhere get NaN, not a + fabricated 0; nse_loss's own masking excludes them naturally. + """ + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Mean of empty slice") + means = np.nanmean(target_tensor, axis=1) + return means.astype(np.float32) + + +def nse_loss(Y_true, Y_pred, per_node_mean, epsilon: float = 1e-6): + """ + 1 - NSE, framed as a loss to minimize. A different signal from + pinball/quantile loss: pinball loss checks whether predicted + quantiles are well-calibrated; NSE checks whether the median + prediction tracks each station's own day-to-day variability, or + degenerates toward a flat, near-mean prediction regardless of + quantile calibration elsewhere. + + Normalized by each station's own historical variance (via + per_node_mean), not a shared global one -- a small, calm stream and + a large, variable one need different absolute error tolerances. + + Applies to the median prediction only, matching every other loss + in this file. + + Args: + Y_true: sparse ground truth, [n_nodes, ...], NaN-masked as + elsewhere in this file. + Y_pred: median prediction, same shape as Y_true. + per_node_mean: [n_nodes], from compute_per_node_historical_mean, + broadcast against Y_true/Y_pred's leading node dimension. + epsilon: prevents division by zero for a node whose historical + variance is near-zero. + + Returns: + Scalar loss (NaN if no real ground truth exists at all). + """ + is_torch = _HAS_TORCH and isinstance(Y_true, torch.Tensor) + + if is_torch: + mean_broadcast = per_node_mean if per_node_mean.dim() == Y_true.dim() else \ + per_node_mean.view(-1, *([1] * (Y_true.dim() - 1))) + mask = ~torch.isnan(Y_true) + if not mask.any(): + return Y_true.sum() * float("nan") + numerator = ((Y_pred - Y_true)[mask] ** 2).sum() + denominator = ((Y_true - mean_broadcast).expand_as(Y_true)[mask] ** 2).sum() + epsilon + return numerator / denominator + + mean_broadcast = per_node_mean if per_node_mean.ndim == Y_true.ndim else \ + per_node_mean.reshape(-1, *([1] * (Y_true.ndim - 1))) + mask = ~np.isnan(Y_true) + if not mask.any(): + return np.nan + numerator = ((Y_pred - Y_true)[mask] ** 2).sum() + denominator = ((Y_true - np.broadcast_to(mean_broadcast, Y_true.shape))[mask] ** 2).sum() + epsilon + return float(numerator / denominator) + + +# --------------------------------------------------------------------------- +# 7. Gated spatial smoothness (Dirichlet energy / graph Laplacian) -- +# only where continuity is actually expected +# --------------------------------------------------------------------------- + +def spatial_smoothness_loss( + Q, edge_index, distance_km, verified_continuous, epsilon: float = 0.1, +): + """ + Dirichlet-energy / graph-Laplacian smoothness term -- predicted + values at two connected, hydrologically-continuous nodes should + vary smoothly along a reach, not erratically. + + Gated on verified_continuous, not applied uniformly graph-wide: La + Risle's bétoire stretch has 3 confirmed karst edges where water + genuinely disappears and can resurge elsewhere, so downstream + values there should differ sharply from upstream ones. Applying + smoothness across those edges would penalize the model for + correctly representing that. They're excluded entirely, not + down-weighted, since even a partial penalty there is still wrong. + + Caveat: verified_continuous currently reflects only what's been + confirmed by station-naming evidence (3 edges). A separate cavity- + proximity cross-check found other stations sitting just as close to + real natural cavities without confirmed karst status -- this gate + protects against what's confirmed discontinuous, not everything + that might be. + + Weighted by inverse distance (1 / (distance_km + epsilon)) -- closer + nodes are expected to behave more similarly than distant ones, so + smoothness is enforced more strongly at short range. + + Args: + Q: predicted discharge, [n_nodes, ...] -- always the model's + own dense prediction, never NaN, so no masking is needed. + edge_index: [2, n_edges] (upstream, downstream) node index pairs. + distance_km: [n_edges] distance per edge. + verified_continuous: [n_edges] boolean/0-1 flag -- edges where + this is False are excluded entirely. + epsilon: prevents an unboundedly large weight for a very small + distance. + + Returns: + Scalar loss (0.0 if no continuous edges are available). + """ + is_torch = _HAS_TORCH and isinstance(Q, torch.Tensor) + + if is_torch: + continuous_mask = verified_continuous.bool() if torch.is_tensor(verified_continuous) else \ + torch.tensor(verified_continuous, dtype=torch.bool, device=Q.device) + if not continuous_mask.any(): + return Q.sum() * 0.0 + up_idx = edge_index[0][continuous_mask] + down_idx = edge_index[1][continuous_mask] + dist = distance_km[continuous_mask] if torch.is_tensor(distance_km) else \ + torch.tensor(np.asarray(distance_km)[continuous_mask.cpu().numpy()], dtype=Q.dtype, device=Q.device) + weights = 1.0 / (dist + epsilon) + diffs_sq = (Q[up_idx] - Q[down_idx]) ** 2 + # weights broadcasts against any extra trailing dims (e.g. a + # horizon axis) the same way distance itself has none of those + # dims -- one real weight per edge, applied identically across + # whatever else Q carries per node. + while weights.dim() < diffs_sq.dim(): + weights = weights.unsqueeze(-1) + weighted = weights * diffs_sq + return weighted.sum() / (weights.sum() * diffs_sq.shape[1:].numel() if diffs_sq.dim() > 1 else weights.sum()) + + continuous_mask = np.asarray(verified_continuous).astype(bool) + if not continuous_mask.any(): + return 0.0 + up_idx = np.asarray(edge_index[0])[continuous_mask] + down_idx = np.asarray(edge_index[1])[continuous_mask] + dist = np.asarray(distance_km)[continuous_mask] + weights = 1.0 / (dist + epsilon) + diffs_sq = (Q[up_idx] - Q[down_idx]) ** 2 + weights_b = weights if diffs_sq.ndim == 1 else weights.reshape(-1, *([1] * (diffs_sq.ndim - 1))) + weighted = weights_b * diffs_sq + return float(weighted.sum() / (np.broadcast_to(weights_b, diffs_sq.shape).sum())) \ No newline at end of file diff --git a/src/graph/pinball_loss.py b/src/graph/pinball_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..70545b499e3195ea16ea5c786e0ca2bb6d5a122d --- /dev/null +++ b/src/graph/pinball_loss.py @@ -0,0 +1,158 @@ +""" +Pinball (quantile) loss -- the actual fix for the failure this project +just measured directly: a model trained on plain MSE caught zero real +flood events at every lead time (0/203 real events, real held-out test +data), not because the architecture was wrong, but because MSE +minimizes AVERAGE error, and real discharge is heavily right-skewed +(confirmed directly against real data: 99th percentile running 4-12x +the median). Predicting close to the mean is a genuinely good strategy +for average error and a genuinely bad one for flood risk. + +Pinball loss for a quantile tau: + L_tau(y, yhat) = max(tau * (y - yhat), (tau - 1) * (y - yhat)) + +This is asymmetric by construction -- for a high tau (e.g. 0.95), +underpredicting is penalized far more than overpredicting by the same +amount, which is exactly the incentive missing from MSE: a model +trained with this loss is pushed toward actually covering the real +tail, not toward the safe average. +""" +from typing import List + +import numpy as np + +try: + import torch + _HAS_TORCH = True +except ImportError: + _HAS_TORCH = False + + +def pinball_loss(y_true, y_pred, quantile: float): + """ + Single-quantile pinball loss, NaN-masked (same principle used + throughout this project's other losses -- a missing real target + must not poison the loss for every other real value in the batch). + + Args: + y_true, y_pred: same shape, any real tensor/array. + quantile: the quantile level this prediction represents, in + (0, 1). + """ + is_np = not _HAS_TORCH or not isinstance(y_true, torch.Tensor) + abs_diff = y_true - y_pred + loss = np.maximum(quantile * abs_diff, (quantile - 1) * abs_diff) if is_np else \ + torch.maximum(quantile * abs_diff, (quantile - 1) * abs_diff) + + is_finite = np.isfinite(loss) if is_np else torch.isfinite(loss) + has_any = is_finite.any() if hasattr(is_finite, "any") else any(is_finite) + if not has_any: + return loss.sum() * float("nan") if not is_np else np.nan + return loss[is_finite].mean() + + +def multi_quantile_loss(y_true, y_pred_quantiles, quantiles: List[float]): + """ + Sums pinball loss across every predicted quantile level -- what a + caller actually uses during training, since the model predicts all + quantiles for a target together, not one at a time. + + Args: + y_true: [*batch_dims] -- the real target, same for every quantile. + y_pred_quantiles: [*batch_dims, n_quantiles] -- one prediction + per quantile level, last axis ordered to match `quantiles`. + quantiles: the quantile levels, e.g. [0.5, 0.9, 0.95, 0.99]. + """ + # Confirmed as a real, persistent bug against real training, not + # hypothetical: y_true.sum() * 0.0 was meant as "a zero of the + # right shape/dtype/device", but y_true (the real target) is + # ~97% NaN by design here -- only a few real gauges ever have real + # discharge/water-level data. A plain (non-nan-aware) sum over a + # mostly-NaN tensor is ALREADY NaN, and NaN * 0.0 is still NaN, not + # zero. total started broken and never recovered, regardless of how + # many real pinball terms the loop below correctly computed -- + # NaN + anything is NaN. This made multi_quantile_loss return NaN + # on every real call in this project, unconditionally, masked until + # now by larger, more visible bugs elsewhere in the same runs. + total = torch.zeros((), dtype=y_true.dtype, device=y_true.device) if _HAS_TORCH and isinstance(y_true, torch.Tensor) \ + else np.zeros((), dtype=y_true.dtype) + n_real_terms = 0 + for i, q in enumerate(quantiles): + term = pinball_loss(y_true, y_pred_quantiles[..., i], q) + is_nan = np.isnan(term) if not _HAS_TORCH or not isinstance(term, torch.Tensor) else torch.isnan(term) + if not is_nan: + total = total + term + n_real_terms += 1 + if n_real_terms == 0: + return total * float("nan") + return total / n_real_terms + + +def weighted_multi_quantile_loss(y_true, y_pred_quantiles, quantiles: List[float], recency_weight, alpha: float = 1.0): + """ + Same real pinball-loss sum as multi_quantile_loss, but each real + element is additionally weighted by 1 + alpha*log(1+dt) before + averaging -- forcing more real loss-weight onto observations that + follow a longer real gap in that node's own history, so the model + is pushed to actually close a real drift accumulated over a gap + rather than being scored the same regardless of how stale its + recent information was. + + A REAL, HONEST OVERLAP WARNING, not a minor footnote: this project + already built and wired in a days-since-last-real-observation + channel as a direct INPUT feature (see dynamic_features.py's + compute_days_since_last_real / compress_days_since_last_real). That + gives the model itself the same underlying real information and + lets it learn how to use staleness; this function instead FORCES + more loss-weight onto post-gap examples regardless of whether the + model finds that useful. Two different mechanisms carrying the same + real underlying signal, in directions that aren't guaranteed to be + aligned -- this is a real, plausible risk of diluting rather than + reinforcing the existing signal, not a hypothetical concern. Treat + alpha=0 (a no-op, recovering exactly multi_quantile_loss's own + behavior) as the real baseline to compare against before trusting + any alpha>0 run's results -- an ablation, not an assumption either + mechanism helps. + + Args: + y_true, y_pred_quantiles, quantiles: same as multi_quantile_loss. + recency_weight: [n_nodes] (or broadcastable against y_true's + leading node dimension) -- real, already-computed + 1 + alpha*log(1+dt) values per node. Passed in already + computed, not recomputed here, so the caller controls + exactly which real dt this reflects (e.g. dt as of the + anchor date, not some other point). + alpha: real scaling on the log(1+dt) term, applied by the + CALLER when building recency_weight -- kept as a parameter + here too only for direct-call convenience; if + recency_weight is passed pre-built, this argument has no + further effect and is not used inside this function. + """ + is_torch = _HAS_TORCH and isinstance(y_true, torch.Tensor) + weight_bcast = recency_weight if (hasattr(recency_weight, "ndim") and recency_weight.ndim == y_true.ndim) else \ + (recency_weight.view(-1, *([1] * (y_true.ndim - 1))) if is_torch else + recency_weight.reshape(-1, *([1] * (y_true.ndim - 1)))) + + total = torch.zeros((), dtype=y_true.dtype, device=y_true.device) if is_torch \ + else np.zeros((), dtype=y_true.dtype) + total_weight = torch.zeros((), dtype=y_true.dtype, device=y_true.device) if is_torch \ + else np.zeros((), dtype=y_true.dtype) + + for i, q in enumerate(quantiles): + y_pred_i = y_pred_quantiles[..., i] + abs_diff = y_true - y_pred_i + raw = torch.maximum(q * abs_diff, (q - 1) * abs_diff) if is_torch else \ + np.maximum(q * abs_diff, (q - 1) * abs_diff) + weighted = raw * (weight_bcast if is_torch else np.broadcast_to(weight_bcast, raw.shape)) + + is_finite = torch.isfinite(weighted) if is_torch else np.isfinite(weighted) + has_any = is_finite.any() if hasattr(is_finite, "any") else any(is_finite) + if not has_any: + continue + w_for_sum = (weight_bcast if is_torch else np.broadcast_to(weight_bcast, raw.shape))[is_finite] + total = total + weighted[is_finite].sum() + total_weight = total_weight + w_for_sum.sum() + + if (total_weight == 0) if not is_torch else (total_weight.item() == 0): + return total * float("nan") + return total / total_weight \ No newline at end of file diff --git a/src/graph/spatiotemporal_gnn.py b/src/graph/spatiotemporal_gnn.py new file mode 100644 index 0000000000000000000000000000000000000000..9dee527f7e70317f3f5593688ef7bb1115bcac14 --- /dev/null +++ b/src/graph/spatiotemporal_gnn.py @@ -0,0 +1,484 @@ +""" +The main spatiotemporal model: predicts discharge/water level at ANY +node (gauged or ungauged) for a requested horizon, using the real reach +graph topology + static/dynamic features built throughout this project. + +Architecture, per the strategy discussion: + - Spatial: directed message passing over the real reach graph, + respecting real flow direction. SUM aggregation at each node + (not mean) -- deliberate, not a default left unconsidered: a + confluence's downstream discharge is the SUM of its upstream + branches (see physics_losses.py's confluence_mass_balance_loss), + so the message-passing aggregation itself should reflect that + physical reality, not average it away. + - Temporal: per-node GRU over the lookback window, fed by the + spatial layer's output at each timestep (not raw features + directly) -- matches the "Option A" architecture decided early in + this project (spatial + temporal kept separate) for sample + efficiency given how sparse real labels are. + - Decoder: horizon-conditioned, SHARED weights across both basins, + with a learned basin embedding as an additional input -- one + backbone for the general "how does water move through a river + network" relationships, basin-specific behavior expressed only at + the final decoding stage. Quantile output (Q, H), not a point + estimate -- a plain-MSE-trained earlier version caught 0 of 203 + real flood events, since MSE optimizes toward the distribution's + center, exactly wrong for flood risk. +""" +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd + +try: + import torch + import torch.nn as nn + from torch_geometric.nn import MessagePassing + _HAS_TORCH = True +except ImportError: + _HAS_TORCH = False + nn = None + MessagePassing = object # placeholder base so class definitions below don't NameError at import time + + +# --------------------------------------------------------------------------- +# Data preparation -- pure numpy/pandas, testable without torch +# --------------------------------------------------------------------------- + +def prepare_graph_training_windows( + x_static: np.ndarray, dynamic_tensors: Dict[str, np.ndarray], target_tensor: np.ndarray, + dates: List[pd.Timestamp], lookback_days: int, horizons: List[int], + dt_tensors: Optional[Dict[str, np.ndarray]] = None, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[pd.Timestamp]]: + """ + Builds graph-wide training windows -- genuinely different from + prepare_training_windows in climate_anomaly_model.py, which could + require an ENTIRE lookback window to be real per (independent) cell + and just skip incomplete ones. That doesn't work at graph scale: + only 6-8 of ~4,500 nodes have ANY real discharge at all, so + requiring every node's window to be fully real would throw away + virtually the entire graph on every single anchor date. + + Instead: missing dynamic values are filled with 0 (after the + caller's own z-scoring, so 0 represents "no information", not a + physically meaningful reading) and paired with an explicit + missingness channel per variable -- same __was_missing principle + already used throughout node_features.py's static features, now + applied per-timestep. The model sees BOTH the (filled) value and + whether it was real, and message passing is what's actually + supposed to let real information at gauged nodes propagate outward + to fill the gap physically, rather than the raw input pretending + a filled zero is a real observation. + + dt_tensors, when provided, adds a THIRD channel per variable: real, + log-compressed days-since-last-real-observation (see + dynamic_features.py's compute_days_since_last_real). A binary + missing flag alone can't distinguish a single-day gap from a + months-long one; this is the fix. Must cover every key in + dynamic_tensors if provided at all -- a partial dt_tensors dict + would silently give some variables 2 channels and others 3, + genuinely breaking the interleaving downstream code relies on, so + this is enforced explicitly rather than left to fail confusingly + later. + + Args: + x_static: [n_nodes, n_static_features] -- constant across the + whole window (graph structure/static features don't change + over the lookback period). + dynamic_tensors: {variable_name: [n_nodes, T]} -- e.g. from + dynamic_features.py's assemble_dynamic_tensor, one entry + per dynamic variable (discharge, groundwater level, etc.) + target_tensor: [n_nodes, T] -- the real quantity being + predicted (discharge or water level), same real/NaN + structure as the dynamic tensors (only real gauges with + real observations have a value). + dates: the T dates target_tensor/dynamic_tensors are indexed by. + lookback_days, horizons: as elsewhere in this project. + dt_tensors: optional {variable_name: [n_nodes, T]}, already + log-compressed, matching dynamic_tensors' own keys exactly. + + Returns: + X_dynamic: [n_examples, lookback_days, n_nodes, C] -- interleaved + per dynamic variable, (value, was_missing) if dt_tensors is + None, (value, was_missing, days_since_real) if provided -- + C = 2*n_dynamic_vars or 3*n_dynamic_vars accordingly. + Y: [n_examples, n_nodes, n_horizons] -- target values, NaN where + no real observation exists (real gauges only, at whatever + horizons they actually have data for). + Y_mask: [n_examples, n_nodes, n_horizons] boolean, True where Y + is real -- returned explicitly rather than relying on + np.isnan downstream, since the loss function needs exactly + this same mask and computing it twice invites the two + copies drifting apart. + anchor_dates: the "today" date for each example. + """ + var_names = list(dynamic_tensors.keys()) + if dt_tensors is not None: + missing_dt_vars = set(var_names) - set(dt_tensors.keys()) + if missing_dt_vars: + raise ValueError(f"dt_tensors is missing entries for {missing_dt_vars} -- must cover " + f"every key in dynamic_tensors, or be None entirely, not a partial set.") + channels_per_var = 3 if dt_tensors is not None else 2 + n_nodes = x_static.shape[0] + n_dates = len(dates) + max_horizon = max(horizons) + + X_list, Y_list, anchor_dates = [], [], [] + + for t in range(lookback_days, n_dates - max_horizon): + window_channels = [] + for var in var_names: + raw_window = dynamic_tensors[var][:, t - lookback_days:t] # [n_nodes, lookback] + missing = np.isnan(raw_window) + filled = np.where(missing, 0.0, raw_window) + window_channels.append(filled) + window_channels.append(missing.astype(np.float32)) + if dt_tensors is not None: + dt_window = dt_tensors[var][:, t - lookback_days:t] + window_channels.append(dt_window) + # stack into [lookback, n_nodes, channels_per_var*n_vars] + stacked = np.stack(window_channels, axis=-1).transpose(1, 0, 2) + + targets = np.stack([target_tensor[:, t + h - 1] for h in horizons], axis=-1) # [n_nodes, n_horizons] + if np.isnan(targets).all(): + continue # no real supervision anywhere in the graph for this anchor date -- physics losses + # still apply at training time even without this window, so skipping here only affects + # the supervised-loss examples, not physics-loss coverage (which the training loop applies + # to every anchor date's model output, not just ones with real targets) + + X_list.append(stacked) + Y_list.append(targets) + anchor_dates.append(dates[t]) + + if not X_list: + return (np.empty((0, lookback_days, n_nodes, channels_per_var * len(var_names))), + np.empty((0, n_nodes, len(horizons))), + np.empty((0, n_nodes, len(horizons)), dtype=bool), []) + + X = np.stack(X_list) + Y = np.stack(Y_list) + Y_mask = ~np.isnan(Y) + return X, Y, Y_mask, anchor_dates + + +# --------------------------------------------------------------------------- +# Model -- requires torch + torch_geometric, NOT executed in this +# environment. Syntax-checked only. +# --------------------------------------------------------------------------- + +if _HAS_TORCH: + class RiverMessagePassing(MessagePassing): + """ + One round of message passing over the real reach graph. + aggr='add' (SUM), not 'mean' -- deliberate: a confluence's + downstream value should reflect the SUM of what flows in from + each upstream branch, matching physics_losses.py's + confluence_mass_balance_loss directly, not an average that + would understate a real confluence's combined inflow. + + Edge features (distance_km, elevation_drop_m, + verified_continuous) condition the message, not just the + source node's features -- a longer, steeper edge should + propagate differently than a short, flat one, and + verified_continuous=False (the bétoire stretch) is exactly the + kind of signal that should suppress how strongly water is + "assumed" to pass through that edge. + """ + + def __init__(self, in_channels: int, out_channels: int, edge_dim: int): + super().__init__(aggr="add") + self.edge_mlp = nn.Sequential(nn.Linear(edge_dim, out_channels), nn.ReLU()) + self.node_mlp = nn.Linear(in_channels, out_channels) + # update() concatenates the ORIGINAL input x (width=in_channels) + # with aggr_out (width=out_channels) -- this must be sized + # in_channels + out_channels, not out_channels * 2. Confirmed as + # a real, not hypothetical, bug: out_channels*2 happened to be + # correct for spatial2 (in_channels == out_channels == hidden_dim + # there, by coincidence) but broke immediately on real data for + # spatial1, where in_channels (static_dim+dynamic_dim=7) != + # out_channels (hidden_dim=64) -- real error: + # "mat1 and mat2 shapes cannot be multiplied (4525x71 and 128x64)", + # 71 = 7 + 64, the actual concatenated width, not 128 = 64*2. + self.update_mlp = nn.Sequential( + nn.Linear(in_channels + out_channels, out_channels), nn.ReLU(), + ) + + def forward(self, x: "torch.Tensor", edge_index: "torch.Tensor", edge_attr: "torch.Tensor") -> "torch.Tensor": + return self.propagate(edge_index, x=x, edge_attr=edge_attr) + + def message(self, x_j: "torch.Tensor", edge_attr: "torch.Tensor") -> "torch.Tensor": + edge_feat = self.edge_mlp(edge_attr) + return self.node_mlp(x_j) * edge_feat + + def update(self, aggr_out: "torch.Tensor", x: "torch.Tensor") -> "torch.Tensor": + return self.update_mlp(torch.cat([x, aggr_out], dim=-1)) + + + class TypeAwareInputEncoder(nn.Module): + """ + Gauge and non-gauge nodes get genuinely separate input + processing -- not the same shared encoder applied to a 0-filled + fabrication for nodes that never had the data. Two encoders: + + - shared_encoder: processes features every node genuinely has + (elevation, IDPR, cumulative catchment area) -- computed + identically for every node, no type distinction needed here + because the data itself isn't type-specific. + - gauge_encoder: processes features that only exist for real + gauges (landcover, geology, NDVI, cavite proximity, real + discharge/groundwater history). Technically evaluated on + EVERY node's input tensor (including non-gauge nodes' 0-filled + placeholders, for simple batched tensor shapes), but the + result is then REPLACED for non-gauge nodes by a single + learned placeholder embedding via torch.where -- gradients + only flow to gauge_encoder's parameters through real gauge + nodes (torch.where's backward routes gradient to whichever + branch was actually selected), so gauge-only parameters never + get spurious updates from fabricated non-gauge inputs, even + though the forward pass computes (and discards) that encoding + for every node. This replaces the model having to LEARN to + discount a "0 + missingness flag" signal with an architectural + guarantee that non-gauge nodes never see gauge-encoder output + presented as if it were real. + """ + + def __init__(self, shared_static_dim: int, gauge_input_dim: int, embed_dim: int): + super().__init__() + self.shared_encoder = nn.Linear(shared_static_dim, embed_dim) + self.gauge_encoder = nn.Sequential(nn.Linear(gauge_input_dim, embed_dim), nn.ReLU()) + self.non_gauge_placeholder = nn.Parameter(torch.zeros(embed_dim)) + + def forward( + self, x_shared_static: "torch.Tensor", x_gauge_static: "torch.Tensor", + x_gauge_dynamic_t: "torch.Tensor", is_gauged: "torch.Tensor", + ) -> "torch.Tensor": + """ + Args: + x_shared_static: [n_nodes, shared_static_dim] -- constant across the window. + x_gauge_static: [n_nodes, gauge_static_dim] -- constant across the window, + real only for gauge nodes (0-filled elsewhere). + x_gauge_dynamic_t: [n_nodes, gauge_dynamic_dim] -- THIS timestep's + gauge-only dynamic history (real only for gauge nodes). + is_gauged: [n_nodes] bool tensor. + + Returns: [n_nodes, embed_dim] combined node representation for this timestep. + """ + shared_embed = self.shared_encoder(x_shared_static) + + gauge_input = torch.cat([x_gauge_static, x_gauge_dynamic_t], dim=-1) + gauge_embed_computed = self.gauge_encoder(gauge_input) + gauge_embed = torch.where( + is_gauged.unsqueeze(-1), gauge_embed_computed, + self.non_gauge_placeholder.unsqueeze(0).expand_as(gauge_embed_computed), + ) + + return shared_embed + gauge_embed + + + class SpatiotemporalGNN(nn.Module): + """ + Full model: type-aware input encoding (see TypeAwareInputEncoder + above), two RiverMessagePassing layers per timestep (two hops -- + lets a node see two edges' worth of upstream influence per step, + e.g. a confluence's own upstream confluence), a per-node GRU + across the lookback window, and a horizon+basin-conditioned + decoder with shared weights across both basins. + """ + + def __init__( + self, shared_static_dim: int, gauge_static_dim: int, gauge_dynamic_dim: int, + edge_dim: int = 5, embed_dim: int = 32, hidden_dim: int = 64, gru_hidden: int = 64, + n_basins: int = 2, basin_embed_dim: int = 8, horizon_embed_dim: int = 8, n_targets: int = 2, + quantiles: Optional[List[float]] = None, forecast_embed_dim: int = 8, + ): + super().__init__() + self.quantiles = quantiles or [0.5, 0.9, 0.95, 0.99] + n_quantiles = len(self.quantiles) + self.input_encoder = TypeAwareInputEncoder( + shared_static_dim, gauge_static_dim + gauge_dynamic_dim, embed_dim, + ) + self.spatial1 = RiverMessagePassing(embed_dim, hidden_dim, edge_dim) + self.spatial2 = RiverMessagePassing(hidden_dim, hidden_dim, edge_dim) + self.temporal = nn.GRU(hidden_dim, gru_hidden, batch_first=True) + self.basin_embed = nn.Embedding(n_basins, basin_embed_dim) + self.horizon_embed = nn.Linear(1, horizon_embed_dim) + # Real forecasted precipitation, per node per horizon -- NOT + # part of the temporal encoder, deliberately: a forecast is + # future-looking information tied to a SPECIFIC horizon (the + # forecast for "3 days out" and "30 days out" are genuinely + # different numbers), whereas the encoder only ever sees one + # shared lookback window of the PAST, applied identically + # across every horizon in a single forward pass. Feeding a + # horizon-specific future value into that shared-history + # pathway would misrepresent it as something it isn't. The + # decoder already conditions per-horizon (via horizon_embed), + # so this is where per-horizon future information belongs. + # + # value+missingness pair, same pattern used throughout this + # project for every other real-but-sometimes-absent input -- + # forward() defaults this to "fully missing" when no real + # forecast tensor is passed in, so this is backward + # compatible with every existing training run and doesn't + # require the real forecast dataset to exist yet. + self.forecast_embed = nn.Sequential(nn.Linear(2, forecast_embed_dim), nn.ReLU()) + self.decoder = nn.Sequential( + nn.Linear(gru_hidden + basin_embed_dim + horizon_embed_dim + forecast_embed_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, n_targets * n_quantiles), + ) + self.n_targets = n_targets + # Manning's roughness -- no real measured value in this + # project's data, learned as a single shared scalar. + # Sigmoid-reparameterized (not log-parameterized) so n is + # ALWAYS within [0.01, 0.3] by construction -- replaces an + # earlier log-parameterization + hard clamp that had a + # confirmed, real failure mode: torch.clamp's gradient is + # exactly zero outside its range (verified directly via + # finite differences), so once the raw parameter drifted + # past the clamp boundary, nothing could ever pull it back. + # Confirmed as the real cause of n landing at 0.38-1.0 + # across several real training runs, never converging. + # manning_n_logit=-2.60269 reproduces the same n=0.03 + # starting point the old log-parameterization used. + self.manning_n_logit = nn.Parameter(torch.tensor(-2.60269)) + # Bank height for the Manning's relaxation gate -- no real + # measurement exists, initialized at 2.5m as a physically + # reasonable starting guess for these rivers (not derived + # from data), same log-positivity trick manning_n_logit + # used to use. Left as-is for now: real training runs show + # this consistently landing at 0.10m, not drifting the way + # n did -- no confirmed evidence this one needs the same fix. + self.log_bank_height_m = nn.Parameter(torch.log(torch.tensor(2.5))) + + def forward( + self, x_shared_static: "torch.Tensor", x_gauge_static: "torch.Tensor", + x_gauge_dynamic_seq: "torch.Tensor", is_gauged: "torch.Tensor", + edge_index: "torch.Tensor", edge_attr: "torch.Tensor", + basin_id: "torch.Tensor", horizons: "torch.Tensor", + forecast_precip: Optional["torch.Tensor"] = None, + forecast_missing: Optional["torch.Tensor"] = None, + ) -> "torch.Tensor": + """ + Args: + x_shared_static: [n_nodes, shared_static_dim] -- constant across the window. + x_gauge_static: [n_nodes, gauge_static_dim] -- constant across the window. + x_gauge_dynamic_seq: [lookback, n_nodes, gauge_dynamic_dim] -- varies per + timestep (lookback-first, matching prepare_graph_training_windows's + real output shape -- see the note lower in this docstring about why + that axis order specifically was worth calling out here before). + is_gauged: [n_nodes] bool tensor. + edge_index, edge_attr: real graph structure, same for every timestep. + basin_id: [n_nodes] long tensor, 0 or 1. + horizons: [n_horizons] -- requested lead times. + forecast_precip: [n_nodes, n_horizons], optional -- real + forecasted precipitation for each node at each + requested horizon (e.g. from a real ECMWF forecast, + mapped station-to-grid-cell the same way the climate + work already does). None (the default) means no real + forecast dataset has been wired in yet -- the model + falls back to treating every value as missing, which + is exactly today's behavior, so existing training + runs are unaffected by this parameter existing. + forecast_missing: [n_nodes, n_horizons], optional -- + 1.0 where forecast_precip has no real value, 0.0 + where it does. Required if forecast_precip is given + (real missingness must be explicit, not inferred), + ignored if forecast_precip is None. + + Returns: + [n_nodes, n_horizons, n_targets, n_quantiles] forecast -- + monotonically increasing along the last axis by + construction (see below), not just intended to be. + """ + lookback = x_gauge_dynamic_seq.shape[0] + embeddings = [] + for t in range(lookback): + x_t = self.input_encoder(x_shared_static, x_gauge_static, x_gauge_dynamic_seq[t, :, :], is_gauged) + h = self.spatial1(x_t, edge_index, edge_attr) + h = self.spatial2(h, edge_index, edge_attr) + embeddings.append(h) + seq = torch.stack(embeddings, dim=1) # [n_nodes, lookback, hidden_dim] + + _, h_n = self.temporal(seq) + node_hidden = h_n.squeeze(0) # [n_nodes, gru_hidden] + + basin_feat = self.basin_embed(basin_id) # [n_nodes, basin_embed_dim] + + n_nodes = node_hidden.shape[0] + n_horizons = horizons.shape[0] + node_hidden_exp = node_hidden.unsqueeze(1).expand(n_nodes, n_horizons, -1) + basin_feat_exp = basin_feat.unsqueeze(1).expand(n_nodes, n_horizons, -1) + horizon_feat = self.horizon_embed( + horizons.float().view(1, n_horizons, 1).expand(n_nodes, n_horizons, 1) + ) + + if forecast_precip is None: + # No real forecast dataset wired in yet -- fully missing, + # matching today's behavior exactly (see docstring). + forecast_precip = torch.zeros(n_nodes, n_horizons, device=node_hidden.device) + forecast_missing = torch.ones(n_nodes, n_horizons, device=node_hidden.device) + forecast_input = torch.stack([forecast_precip, forecast_missing], dim=-1) # [n_nodes, n_horizons, 2] + forecast_feat = self.forecast_embed(forecast_input) + + decoder_input = torch.cat([node_hidden_exp, basin_feat_exp, horizon_feat, forecast_feat], dim=-1) + raw = self.decoder(decoder_input) # [n_nodes, n_horizons, n_targets * n_quantiles] + + n_quantiles = len(self.quantiles) + raw = raw.view(n_nodes, n_horizons, self.n_targets, n_quantiles) + + # Raw decoder output bounded before the monotonicity + # transform -- an untrained model's raw output is + # essentially unconstrained noise (no output activation + # bounds it), and that noise now feeds pinball loss + # directly. Confirmed as a real, not hypothetical, gap: + # Manning's equation got exactly this kind of clamp (on + # depth, n, hydraulic_radius) after a real training run + # produced non-finite validation loss, but that same + # reasoning was never applied to the decoder's own output, + # which is upstream of every loss in this model, not just + # Manning's. +-50 in standardized units is a generous + # ceiling -- real discharge only ever reached ~18x its + # median even at extreme floods, which standardizes to well + # under this bound -- so this loses no real information + # while giving a hard numerical floor under everything + # downstream. + raw = torch.clamp(raw, min=-50.0, max=50.0) + + # Monotonicity across quantiles is architecturally guaranteed, + # not hoped for -- a real, well-known failure mode of naive + # independently-trained quantile regression is "quantile + # crossing" (e.g. the predicted 90th percentile coming out + # lower than the predicted median), which is physically + # nonsensical for a real cumulative distribution. The lowest + # quantile (self.quantiles[0], e.g. the median) is predicted + # directly and unconstrained; every subsequent quantile is + # the previous one plus a softplus-transformed offset, which + # is always >= 0 by construction -- so quantiles[k] >= + # quantiles[k-1] for every k, always, regardless of what + # gradient descent does to the underlying raw outputs. + first = raw[..., 0:1] + increments = torch.nn.functional.softplus(raw[..., 1:]) + quantile_preds = torch.cat([first, first + torch.cumsum(increments, dim=-1)], dim=-1) + return quantile_preds + + + def masked_mse_graph(pred: "torch.Tensor", target: "torch.Tensor", mask: "torch.Tensor") -> "torch.Tensor": + """ + Supervised loss, masked to real observations only -- reusing + the exact NaN-masking principle already validated for + physics_losses.py and the climate anomaly model's loss, applied + here to the [n_nodes, n_horizons, n_targets] prediction shape. + Real gauges only ever supply a tiny fraction of [n_nodes, + n_horizons] cells; everywhere else this loss contributes + nothing, and only the physics losses (evaluated separately, + graph-wide, on the same forward pass's output) provide training + signal. + """ + if not mask.any(): + return pred.sum() * float("nan") + return ((pred[mask] - target[mask]) ** 2).mean() + +else: + RiverMessagePassing = None + SpatiotemporalGNN = None + masked_mse_graph = None \ No newline at end of file diff --git a/src/graph/subgraph_selection.py b/src/graph/subgraph_selection.py new file mode 100644 index 0000000000000000000000000000000000000000..15f701384e1f18ba8ea17aadbe2dc49a089885ce --- /dev/null +++ b/src/graph/subgraph_selection.py @@ -0,0 +1,159 @@ +""" +Selects a small subgraph -- real gauges plus a capped number of real +confluences/braid nodes -- for fast training iteration, without losing +real connectivity the way a naive node filter would. + +THE REAL PROBLEM THIS SOLVES: most edges between two gauges (or a gauge +and a confluence) pass through several virtual infill nodes. Simply +dropping every non-selected node and keeping only edges where BOTH +endpoints survive would leave most selected nodes disconnected from +each other -- their only real path ran through the nodes just removed. +That would break both message passing (no path for information to +flow) and the physics losses (routing_consistency_loss needs a real +lag between connected nodes; confluence_mass_balance_loss needs real +upstream branches). + +THE FIX: for each dropped chain of virtual nodes between two kept +nodes, collapse it into a single new edge, summing distance_km and +elevation_drop_m along the real path, and ANDing verified_continuous +(if any real segment along the collapsed path was flagged +non-continuous -- e.g. the bétoire karst stretch -- the collapsed edge +correctly inherits that, rather than silently losing the flag). +""" +from typing import List, Optional, Set, Tuple + +import numpy as np +import pandas as pd + + +def select_subgraph_nodes( + nodes_df: pd.DataFrame, max_nodes: int = 100, + include_confluences: bool = True, max_braid_nodes: Optional[int] = None, +) -> Set[str]: + """ + Real gauges first (never dropped, whatever the budget), then real + confluences (small in number system-wide -- 28 across both basins + in the last real reach-graph build -- so "include all" is a + reasonable default, not likely to blow the budget on its own), then + braid (split/rejoin) nodes filling whatever budget remains. + + Returns the set of station_codes to keep. + """ + gauge_codes = set(nodes_df.loc[nodes_df["is_gauged"], "station_code"]) + if len(gauge_codes) > max_nodes: + raise ValueError(f"{len(gauge_codes)} real gauge(s) alone exceed max_nodes={max_nodes} -- " + f"raise max_nodes, gauges are never dropped.") + + selected = set(gauge_codes) + remaining_budget = max_nodes - len(selected) + + if include_confluences and remaining_budget > 0 and "is_confluence" in nodes_df.columns: + confluence_codes = nodes_df.loc[nodes_df["is_confluence"], "station_code"].tolist() + take = confluence_codes[:remaining_budget] + selected.update(take) + remaining_budget -= len(take) + + if remaining_budget > 0 and "is_split_point" in nodes_df.columns and "is_rejoin_point" in nodes_df.columns: + braid_codes = nodes_df.loc[ + nodes_df["is_split_point"] | nodes_df["is_rejoin_point"], "station_code" + ].tolist() + n_braid_take = min(remaining_budget, max_braid_nodes) if max_braid_nodes else remaining_budget + selected.update(braid_codes[:n_braid_take]) + + return selected + + +def _build_adjacency(edges_df: pd.DataFrame) -> dict: + """station_code -> list of (neighbor_code, distance_km, elevation_drop_m, verified_continuous, width_m)""" + adj = {} + has_width = "width_m" in edges_df.columns + for _, e in edges_df.iterrows(): + width = e["width_m"] if has_width else np.nan + adj.setdefault(e["source"], []).append( + (e["target"], e["distance_km"], e["elevation_drop_m"], bool(e.get("verified_continuous", True)), width) + ) + return adj + + +def _collapse_path_from( + start: str, adj: dict, selected: Set[str], max_hops: int = 500, +) -> List[Tuple[str, float, float, bool, float]]: + """ + Walks forward from `start` along real edges until hitting the next + selected node (or a dead end), summing distance/elevation along the + way. Returns a list of (reached_selected_node, total_distance_km, + total_elevation_drop_m, all_verified_continuous, avg_width_m) -- a + list, not a single result, because a node can have multiple + outgoing edges (a real split point), each leading to a different + eventual selected node. + + Width is handled differently from distance/elevation deliberately: + it is NOT additive -- a collapsed edge spanning several real + tronçons has a genuine distance-weighted AVERAGE width, not a sum + (summing would make a long collapsed edge look absurdly wide). + Segments with no real width data (width_m is NaN -- see + compute_edge_width.py, which leaves it NaN rather than fabricating + a value when no real BD TOPO surface polygon matched) are excluded + from the average, weighted by their own real distance; the + collapsed edge's width only falls back to NaN if NONE of its real + segments had width data at all, the same "skip missing, don't + fabricate" principle used in compute_edge_width.py's own + aggregation. + """ + results = [] + # (node, dist, elev, cont, hops, width_weighted_sum, width_weight_total) + stack = [(start, 0.0, 0.0, True, 0, 0.0, 0.0)] + visited_this_walk = set() + + while stack: + node, dist, elev, cont, hops, w_sum, w_weight = stack.pop() + if hops > max_hops: + continue # defensive -- avoid infinite loops on a real but unexpected cycle + for neighbor, seg_dist, seg_elev, seg_cont, seg_width in adj.get(node, []): + new_dist, new_elev, new_cont = dist + seg_dist, elev + seg_elev, cont and seg_cont + if not np.isnan(seg_width): + new_w_sum, new_w_weight = w_sum + seg_width * seg_dist, w_weight + seg_dist + else: + new_w_sum, new_w_weight = w_sum, w_weight + + if neighbor in selected: + avg_width = new_w_sum / new_w_weight if new_w_weight > 0 else float("nan") + results.append((neighbor, new_dist, new_elev, new_cont, avg_width)) + elif (neighbor, hops + 1) not in visited_this_walk: + visited_this_walk.add((neighbor, hops + 1)) + stack.append((neighbor, new_dist, new_elev, new_cont, hops + 1, new_w_sum, new_w_weight)) + + return results + + +def build_collapsed_subgraph( + nodes_df: pd.DataFrame, edges_df: pd.DataFrame, max_nodes: int = 100, + include_confluences: bool = True, max_braid_nodes: Optional[int] = None, +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """ + The real deliverable: a small nodes_df/edges_df pair, real + connectivity preserved via collapsed edges, ready to feed directly + into combine_basins the same way the full reach graph is. + """ + selected = select_subgraph_nodes(nodes_df, max_nodes, include_confluences, max_braid_nodes) + small_nodes_df = nodes_df[nodes_df["station_code"].isin(selected)].reset_index(drop=True) + + adj = _build_adjacency(edges_df) + collapsed_rows = [] + seen_pairs = set() # avoid duplicate (source, target) edges from multiple equivalent paths + for source in selected: + for target, dist, elev, cont, width in _collapse_path_from(source, adj, selected): + if source == target: + continue # a path that loops back to its own start isn't a real edge + key = (source, target) + if key in seen_pairs: + continue + seen_pairs.add(key) + collapsed_rows.append({ + "source": source, "target": target, + "distance_km": dist, "elevation_drop_m": elev, "verified_continuous": cont, + "width_m": width, + }) + + small_edges_df = pd.DataFrame(collapsed_rows) + return small_nodes_df, small_edges_df \ No newline at end of file diff --git a/src/logs.txt b/src/logs.txt new file mode 100644 index 0000000000000000000000000000000000000000..cca05fc3b4755ad50f4a08696be313bad577e802 --- /dev/null +++ b/src/logs.txt @@ -0,0 +1,770 @@ +(.venv) macbook@austen:~/Desktop/miradi/gnn/PoC_v1 python -m scripts.train_spatiotemporal_gnn --data-root datasets +eure: 2883 nodes, 3358 edges +risle: 1642 nodes, 1873 edges +Combined graph: 4525 nodes, 5231 edges +Real discharge coverage: 8923 / 23136325 +Standardized 'discharge': train-period mean=2809.26, std=3286.51 (fit on real training-period values only) +Standardized 'groundwater_level': train-period mean=102.62, std=39.37 (fit on real training-period values only) +Physics indices: 28 confluences, 696 braid pairs, 5231 routing edges +Training windows: X=(4903, 30, 4525, 4), Y=(4903, 4525, 10) + +Train examples: 3257, val: 730, test: 916 +Using device: cpu +Physics loss weights: {'confluence': 1.0, 'split_rejoin': 1.0, 'routing': 2.0} + +epoch 1/100: train loss = 0.8412, val supervised loss = 1.1929 (best) +epoch 2/100: train loss = 0.7969, val supervised loss = 1.1486 (best) +epoch 3/100: train loss = 0.7908, val supervised loss = 1.1795 +epoch 4/100: train loss = 0.7955, val supervised loss = 1.1162 (best) +epoch 5/100: train loss = 0.7808, val supervised loss = 1.1066 (best) +epoch 6/100: train loss = 0.7651, val supervised loss = 1.1549 +epoch 7/100: train loss = 0.7411, val supervised loss = 1.0764 (best) +epoch 8/100: train loss = 0.7027, val supervised loss = 0.9924 (best) +epoch 9/100: train loss = 0.6924, val supervised loss = 1.3795 +epoch 10/100: train loss = 0.6889, val supervised loss = 1.0943 +epoch 11/100: train loss = 0.6836, val supervised loss = 1.0572 +epoch 12/100: train loss = 0.6827, val supervised loss = 0.9846 (best) +epoch 13/100: train loss = 0.6763, val supervised loss = 0.9921 +epoch 14/100: train loss = 0.6743, val supervised loss = 1.0505 +epoch 15/100: train loss = 0.6750, val supervised loss = 1.0760 +epoch 16/100: train loss = 0.6751, val supervised loss = 1.0425 +epoch 17/100: train loss = 0.6696, val supervised loss = 0.9860 +epoch 18/100: train loss = 0.6670, val supervised loss = 1.0494 +epoch 19/100: train loss = 0.6643, val supervised loss = 1.0208 +epoch 20/100: train loss = 0.6631, val supervised loss = 1.0441 +epoch 21/100: train loss = 0.6619, val supervised loss = 1.0921 +epoch 22/100: train loss = 0.6603, val supervised loss = 1.0351 +stopped early at epoch 22 + +Best validation loss: 0.9846 +Saved model to datasets/spatiotemporal_gnn/model.pt + + +eure: 2883 real nodes, 3358 real edges (full reach graph) +eure: reduced to 50 node(s) (real gauges + confluences/braids only, no virtual infill nodes), 67 collapsed edge(s), real connectivity preserved +risle: 1642 real nodes, 1873 real edges (full reach graph) +risle: reduced to 50 node(s) (real gauges + confluences/braids only, no virtual infill nodes), 86 collapsed edge(s), real connectivity preserved +Combined graph: 100 nodes, 153 edges +Real discharge coverage: 8923 / 511300 +Standardized 'discharge': train-period mean=2809.26, std=3286.51 (fit on real training-period values only) +Standardized 'groundwater_level': train-period mean=106.01, std=39.00 (fit on real training-period values only) +Physics indices: 11 confluences, 1 braid pairs, 153 routing edges +Training windows: X=(4903, 30, 100, 4), Y=(4903, 100, 10) +Train examples: 3257, val: 730, test: 916 +Using device: cpu +Physics loss weights: {'confluence': 1.0, 'split_rejoin': 1.0, 'routing': 2.0} +epoch 1/100: train loss = 1.1519, val supervised loss = 1.4610 (best) +epoch 2/100: train loss = 0.7972, val supervised loss = 1.3024 (best) +epoch 3/100: train loss = 0.7899, val supervised loss = 1.2201 (best) +epoch 4/100: train loss = 0.7846, val supervised loss = 1.2223 +epoch 5/100: train loss = 0.7906, val supervised loss = 1.2654 +epoch 6/100: train loss = 0.8115, val supervised loss = 1.1744 (best) +epoch 7/100: train loss = 0.7939, val supervised loss = 1.2215 +epoch 8/100: train loss = 0.7976, val supervised loss = 1.2869 +epoch 9/100: train loss = 0.8045, val supervised loss = 1.2263 +epoch 10/100: train loss = 0.7929, val supervised loss = 1.3447 +epoch 11/100: train loss = 0.7906, val supervised loss = 1.2693 +epoch 12/100: train loss = 0.7902, val supervised loss = 1.2461 +epoch 13/100: train loss = 0.7915, val supervised loss = 1.1545 (best) +epoch 14/100: train loss = 0.7886, val supervised loss = 1.2151 +epoch 15/100: train loss = 0.8031, val supervised loss = 1.1819 +epoch 16/100: train loss = 0.7914, val supervised loss = 1.1730 +epoch 17/100: train loss = 0.7864, val supervised loss = 1.1729 +epoch 18/100: train loss = 0.7863, val supervised loss = 1.1727 +epoch 19/100: train loss = 0.7858, val supervised loss = 1.3309 +epoch 20/100: train loss = 0.7856, val supervised loss = 1.2335 +epoch 21/100: train loss = 0.7862, val supervised loss = 1.1961 +epoch 22/100: train loss = 0.7830, val supervised loss = 1.1632 +epoch 23/100: train loss = 0.7835, val supervised loss = 1.2377 +stopped early at epoch 23 +Best validation loss: 1.1545 +Saved model to datasets/spatiotemporal_gnn/model.pt +Naive baseline (predict the training-period mean, i.e. 0 in standardized units): 2.5756 +Model beats the naive baseline by 55.2% -- real, not just baseline-matching skill + +(.venv) macbook@austen:~/Desktop/miradi/gnn/PoC_v1 python -m scripts.train_spatiotemporal_gnn --data-root datasets --train-end 2025-06-30 --val-end 2025-12-31 + eure: joined real channel width for 2805/3358 edge(s) from datasets/edge_widths.csv +eure: 2883 real nodes, 3358 real edges (full reach graph) +eure: reduced to 50 node(s) (real gauges + confluences/braids only, no virtual infill nodes), 67 collapsed edge(s), real connectivity preserved + risle: joined real channel width for 1488/1873 edge(s) from datasets/edge_widths.csv +risle: 1642 real nodes, 1873 real edges (full reach graph) +risle: reduced to 50 node(s) (real gauges + confluences/braids only, no virtual infill nodes), 86 collapsed edge(s), real connectivity preserved +Combined graph: 100 nodes, 153 edges +Real discharge coverage: 8923 / 511300 +Real water-level coverage: 21947 / 511300 +Real catchment area available for 16/100 node(s) +Fitted specific-discharge coefficient k=0.0766876 on 2603 real (Q, precip, area) triple(s) from gauged stations, training period only +Real precipitation-based discharge estimate available for 16496 node-date value(s) (input feature, never a target) +Standardized 'discharge': train-period mean=4168.71, std=5311.31 (fit on real training-period values only) +Standardized 'groundwater_level': train-period mean=105.86, std=39.03 (fit on real training-period values only) +Standardized 'precipitation': train-period mean=1.78, std=4.00 (fit on real training-period values only) +Standardized 'estimated_discharge': train-period mean=3659.35, std=7011.12 (fit on real training-period values only) +Standardized 'waterlevel': train-period mean=402.00, std=590.37 (fit on real training-period values only) +Physics indices: 11 confluences, 1 braid pairs, 153 routing edges +Training windows: X=(4903, 30, 100, 8), Y=(4903, 100, 10) +Water-level targets at the same 4903 anchor dates: Y_H=(4903, 100, 10), 215936 real value(s) + +Real forecast precipitation available for 61830 (example, node, horizon) value(s) -- real coverage is partial by design (only horizons [1, 2, 3, 5, 7], only from 2024-03-01 onward) + +Train examples: 4534, val: 184, test: 185 +Using device: cpu +Physics loss weights: {'confluence': 1.0, 'split_rejoin': 1.0, 'routing': 2.0, 'manning': 1.0, 'water_balance': 1.0} + +Real precipitation available -- water balance physics enabled (channels 4/5) + [debug] example (epoch 1, remaining debug budget 24): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 30/1000 + [debug] confluence term: 81.1335, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283629, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=10, max=25, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.449464, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.53244, finite=True + [debug] supervised_H: 0.179025, finite=True + [debug] physics total: 105.943, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 23): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 26/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 81.1334, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283612, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=4, max=13, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.0549348, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.367937, finite=True + [debug] supervised_H: 0.193409, finite=True + [debug] physics total: 105.548, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 22): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 5/1000 + [debug] Y_H_i real value count: 51/1000 + [debug] confluence term: 81.1334, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283658, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=6.6, max=20.8, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.191883, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 1.01059, finite=True + [debug] supervised_H: 0.253062, finite=True + [debug] physics total: 105.685, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 21): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 37/1000 + [debug] confluence term: 81.1335, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.028364, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=8.6, max=19.7, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.204664, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.52279, finite=True + [debug] supervised_H: 0.156358, finite=True + [debug] physics total: 105.698, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 20): + [debug] Q_pred_quantiles: finite_all=True, min=-0.065, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 20/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 81.1334, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283685, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=6.2, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.360339, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.562039, finite=True + [debug] supervised_H: 0.346614, finite=True + [debug] physics total: 105.853, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 19): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 30/1000 + [debug] confluence term: 81.1334, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283629, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=0.4, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.914814, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.589836, finite=True + [debug] supervised_H: 0.211875, finite=True + [debug] physics total: 106.408, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 18): + [debug] Q_pred_quantiles: finite_all=True, min=-0.065, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 20/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 81.1337, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283742, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=10.6, max=21, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.357288, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.548801, finite=True + [debug] supervised_H: 0.431054, finite=True + [debug] physics total: 105.851, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 17): + [debug] Q_pred_quantiles: finite_all=True, min=-0.06501, max=14.41 + [debug] H_pred_quantiles: finite_all=True, min=-0.16, max=5.462 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 22/1000 + [debug] confluence term: 81.1335, finite=True + [debug] split_rejoin term: 0.000716772, finite=True + [debug] manning term: 24.3307, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0283632, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=4314, max=7186, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=9, max=16.5, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.206747, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.586394, finite=True + [debug] supervised_H: 0.193314, finite=True + [debug] physics total: 105.7, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 16): + [debug] Q_pred_quantiles: finite_all=True, min=-0.1888, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 22/1000 + [debug] Y_H_i real value count: 43/1000 + [debug] confluence term: 58.8818, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.032642, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.6, max=2, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.362139, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.540033, finite=True + [debug] supervised_H: 0.188887, finite=True + [debug] physics total: 77.2119, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 15): + [debug] Q_pred_quantiles: finite_all=True, min=-0.189, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 9/1000 + [debug] Y_H_i real value count: 59/1000 + [debug] confluence term: 58.882, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0326606, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=5.4, max=21.4, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.464889, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.478132, finite=True + [debug] supervised_H: 0.300983, finite=True + [debug] physics total: 77.3149, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 14): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2166, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 30/1000 + [debug] Y_H_i real value count: 40/1000 + [debug] confluence term: 58.8827, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.9346, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0341154, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3549, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=1, n_real=20 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.622464, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.525364, finite=True + [debug] supervised_H: 0.151529, finite=True + [debug] physics total: 77.4742, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 13): + [debug] Q_pred_quantiles: finite_all=True, min=-0.1888, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 9/1000 + [debug] Y_H_i real value count: 59/1000 + [debug] confluence term: 58.8818, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0326413, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=2, max=16.7, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.58037, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.521639, finite=True + [debug] supervised_H: 0.299622, finite=True + [debug] physics total: 77.4302, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 12): + [debug] Q_pred_quantiles: finite_all=True, min=-0.1888, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 25/1000 + [debug] Y_H_i real value count: 49/1000 + [debug] confluence term: 58.8818, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0326409, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=4.4, max=12.3, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.240601, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.300028, finite=True + [debug] supervised_H: 0.206646, finite=True + [debug] physics total: 77.0904, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 11): + [debug] Q_pred_quantiles: finite_all=True, min=-0.189, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 14/1000 + [debug] Y_H_i real value count: 47/1000 + [debug] confluence term: 58.882, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.9349, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0326483, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=5.6, max=13.5, n_real=19 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.471311, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.766813, finite=True + [debug] supervised_H: 0.201794, finite=True + [debug] physics total: 77.3212, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 10): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2775, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 29/1000 + [debug] Y_H_i real value count: 46/1000 + [debug] confluence term: 58.8864, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0402923, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3402, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.2, max=6.3, n_real=20 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.522789, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.438725, finite=True + [debug] supervised_H: 0.157988, finite=True + [debug] physics total: 77.3848, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 9): + [debug] Q_pred_quantiles: finite_all=True, min=-0.1889, max=12.57 + [debug] H_pred_quantiles: finite_all=True, min=-0.1493, max=4.844 + [debug] Y_i real value count: 20/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 58.8818, finite=True + [debug] split_rejoin term: 0.000370772, finite=True + [debug] manning term: 17.935, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.032641, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=3637, max=6725, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.4, max=4.6, n_real=13 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.431347, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.478395, finite=True + [debug] supervised_H: 0.352821, finite=True + [debug] physics total: 77.2811, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 8): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 13/1000 + [debug] Y_H_i real value count: 45/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413045, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=2.4, max=5.2, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.174067, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.376802, finite=True + [debug] supervised_H: 0.258603, finite=True + [debug] physics total: 53.6543, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 7): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 29/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413093, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=0.6, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 1, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.350273, finite=True + [debug] supervised_H: 0.168079, finite=True + [debug] physics total: 54.4802, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 6): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2807, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 30/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413074, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=19.7, max=92.5, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.810393, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.380917, finite=True + [debug] supervised_H: 0.157874, finite=True + [debug] physics total: 54.2906, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 5): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 29/1000 + [debug] Y_H_i real value count: 41/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8063, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413193, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=0.6, n_real=20 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.601746, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.353307, finite=True + [debug] supervised_H: 0.20659, finite=True + [debug] physics total: 54.0818, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 4): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 59/1000 + [debug] confluence term: 40.6322, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413075, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.6, max=24.7, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.546189, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.43953, finite=True + [debug] supervised_H: 0.320977, finite=True + [debug] physics total: 54.0264, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 3): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 27/1000 + [debug] Y_H_i real value count: 49/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413097, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=3.6, max=26.3, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.131423, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.359537, finite=True + [debug] supervised_H: 0.187993, finite=True + [debug] physics total: 53.6116, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 2): + [debug] Q_pred_quantiles: finite_all=True, min=-0.2806, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 20/1000 + [debug] Y_H_i real value count: 50/1000 + [debug] confluence term: 40.6323, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0413066, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2999, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.6, max=2.4, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.471807, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.415129, finite=True + [debug] supervised_H: 0.417917, finite=True + [debug] physics total: 53.952, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 1): + [debug] Q_pred_quantiles: finite_all=True, min=-0.3152, max=10.76 + [debug] H_pred_quantiles: finite_all=True, min=-0.1475, max=4.197 + [debug] Y_i real value count: 30/1000 + [debug] Y_H_i real value count: 40/1000 + [debug] confluence term: 40.6364, finite=True + [debug] split_rejoin term: 0.000253812, finite=True + [debug] manning term: 12.8064, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.044186, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2847, max=5719, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=-9.537e-07, max=0.4, n_real=20 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.972517, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.357991, finite=True + [debug] supervised_H: 0.150745, finite=True + [debug] physics total: 54.4598, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] example (epoch 1, remaining debug budget 0): + [debug] Q_pred_quantiles: finite_all=True, min=-0.3686, max=9.093 + [debug] H_pred_quantiles: finite_all=True, min=-0.1423, max=4.156 + [debug] Y_i real value count: 10/1000 + [debug] Y_H_i real value count: 30/1000 + [debug] confluence term: 26.4635, finite=True + [debug] split_rejoin term: 0.000407118, finite=True + [debug] manning term: 8.60421, finite=True + [debug] GRADIENT of confluence term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of split_rejoin term: wrt Q_median finite=True, wrt H_median finite=True + [debug] GRADIENT of manning term: wrt Q_median finite=True, wrt H_median finite=True + [debug] routing term: 0.0488792, finite=True + [debug] GRADIENT of routing term: wrt Q_median finite=True + [debug] Q_short_block_avg (real L/s): min=2472, max=5445, finite_all=True + [debug] recent_precip_sum_mm (real mm): min=0.2, max=1.2, n_real=14 + [debug] catchment_area_km2: n_real=16, min=149, max=5935 + [debug] water_balance term (pre-weight): 0.633285, finite=True + [debug] GRADIENT of water_balance term: wrt Q_median finite=True + [debug] supervised_Q: 0.290417, finite=True + [debug] supervised_H: 0.157283, finite=True + [debug] physics total: 35.7503, finite=True + [debug] GRADIENT of supervised_Q: wrt Q_pred_quantiles finite=True + [debug] GRADIENT of supervised_H: wrt H_pred_quantiles finite=True + [debug] raw accounting: n_batches=567, total_loss=794.492, len(train_loader)=567 +epoch 1/100: train loss = 1.4012, val Q loss = 0.1503, val H loss = 0.0783, val combined = 0.2286 (best) + [debug] raw accounting: n_batches=567, total_loss=420.515, len(train_loader)=567 +epoch 2/100: train loss = 0.7416, val Q loss = 0.1521, val H loss = 0.0698, val combined = 0.2219 (best) + [debug] raw accounting: n_batches=567, total_loss=418.261, len(train_loader)=567 +epoch 3/100: train loss = 0.7377, val Q loss = 0.1614, val H loss = 0.0744, val combined = 0.2358 + [debug] raw accounting: n_batches=567, total_loss=416.014, len(train_loader)=567 +epoch 4/100: train loss = 0.7337, val Q loss = 0.1463, val H loss = 0.0757, val combined = 0.2221 + [debug] raw accounting: n_batches=567, total_loss=418.748, len(train_loader)=567 +epoch 5/100: train loss = 0.7385, val Q loss = 0.1590, val H loss = 0.0767, val combined = 0.2357 + [debug] raw accounting: n_batches=567, total_loss=414.083, len(train_loader)=567 +epoch 6/100: train loss = 0.7303, val Q loss = 0.1702, val H loss = 0.0769, val combined = 0.2471 + [debug] raw accounting: n_batches=567, total_loss=413.883, len(train_loader)=567 +epoch 7/100: train loss = 0.7300, val Q loss = 0.1551, val H loss = 0.0756, val combined = 0.2307 + [debug] raw accounting: n_batches=567, total_loss=412.084, len(train_loader)=567 +epoch 8/100: train loss = 0.7268, val Q loss = 0.1655, val H loss = 0.0748, val combined = 0.2404 + [debug] raw accounting: n_batches=567, total_loss=411.92, len(train_loader)=567 +epoch 9/100: train loss = 0.7265, val Q loss = 0.1620, val H loss = 0.0783, val combined = 0.2403 + [debug] raw accounting: n_batches=567, total_loss=411.669, len(train_loader)=567 +epoch 10/100: train loss = 0.7260, val Q loss = 0.1568, val H loss = 0.0724, val combined = 0.2293 + [debug] raw accounting: n_batches=567, total_loss=412.194, len(train_loader)=567 +epoch 11/100: train loss = 0.7270, val Q loss = 0.1463, val H loss = 0.0748, val combined = 0.2211 (best) + [debug] raw accounting: n_batches=567, total_loss=405.851, len(train_loader)=567 +epoch 12/100: train loss = 0.7158, val Q loss = 0.1439, val H loss = 0.0649, val combined = 0.2088 (best) + [debug] raw accounting: n_batches=567, total_loss=415.095, len(train_loader)=567 +epoch 13/100: train loss = 0.7321, val Q loss = 0.1703, val H loss = 0.0836, val combined = 0.2540 + [debug] raw accounting: n_batches=567, total_loss=416.673, len(train_loader)=567 +epoch 14/100: train loss = 0.7349, val Q loss = 0.1684, val H loss = 0.0829, val combined = 0.2513 + [debug] raw accounting: n_batches=567, total_loss=417.424, len(train_loader)=567 +epoch 15/100: train loss = 0.7362, val Q loss = 0.1841, val H loss = 0.0858, val combined = 0.2699 + [debug] raw accounting: n_batches=567, total_loss=414.205, len(train_loader)=567 +epoch 16/100: train loss = 0.7305, val Q loss = 0.1647, val H loss = 0.0795, val combined = 0.2441 + [debug] raw accounting: n_batches=567, total_loss=411.842, len(train_loader)=567 +epoch 17/100: train loss = 0.7264, val Q loss = 0.1664, val H loss = 0.0793, val combined = 0.2457 + [debug] raw accounting: n_batches=567, total_loss=411.079, len(train_loader)=567 +epoch 18/100: train loss = 0.7250, val Q loss = 0.1611, val H loss = 0.0809, val combined = 0.2420 + [debug] raw accounting: n_batches=567, total_loss=411.693, len(train_loader)=567 +epoch 19/100: train loss = 0.7261, val Q loss = 0.1516, val H loss = 0.0746, val combined = 0.2261 + [debug] raw accounting: n_batches=567, total_loss=411.414, len(train_loader)=567 +epoch 20/100: train loss = 0.7256, val Q loss = 0.1677, val H loss = 0.0796, val combined = 0.2473 + [debug] raw accounting: n_batches=567, total_loss=411.121, len(train_loader)=567 +epoch 21/100: train loss = 0.7251, val Q loss = 0.1586, val H loss = 0.0812, val combined = 0.2398 + [debug] raw accounting: n_batches=567, total_loss=410.599, len(train_loader)=567 +epoch 22/100: train loss = 0.7242, val Q loss = 0.1534, val H loss = 0.0750, val combined = 0.2283 +stopped early at epoch 22 + +Best validation loss: 0.2088 +Learned Manning's roughness n = 0.9932 (0.03 was the initialization; real natural streams typically fall in ~0.025-0.1 -- a value drifting far outside that range is worth investigating, not just noting) +Learned bank height = 0.10m (2.5m was the initialization, an assumption with no real measured data behind it -- compare this against real rating-curve data for the gauges with one, if the model's learned value looks implausible for these specific rivers) +Saved model to datasets/spatiotemporal_gnn/model.pt + +Naive baseline, discharge (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1728 +Naive baseline, water level (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1276 +Discharge: model beats the naive baseline by 16.7% -- real, not just baseline-matching skill +Water level: model beats the naive baseline by 49.1% -- real, not just baseline-matching skill + +(.venv) macbook@austen:~/Desktop/miradi/gnn/PoC_v1 python -m scripts.evaluation.evaluate_flood_detection --data-root datasets --train-end 2025-06-30 --val-end 2025-12-31 +Loaded 125 real event(s) across 6 station(s) + eure: joined real channel width for 2805/3358 edge(s) from datasets/edge_widths.csv + risle: joined real channel width for 1488/1873 edge(s) from datasets/edge_widths.csv +Fitted specific-discharge coefficient k=0.0766876 on 2603 real (Q, precip, area) triple(s) from gauged stations, training period only +Evaluating on 185 real held-out test example(s) +Real forecast precipitation available for 13320 test (example, node, horizon) value(s) +Loaded trained model from datasets/spatiotemporal_gnn/model.pt +Flagging a real event when the model's predicted 0.95 quantile exceeds that station's real threshold -- not the median, since flood risk is a tail question: "is there a real 5% chance this exceeds the threshold", not "what's the single most likely value." + +====================================================================== +Flood-detection confusion matrix by lead time (real held-out test data) +====================================================================== + horizon_days tp fp fn tn precision recall + 1 15 351 23 721 0.040984 0.394737 + 2 17 381 21 691 0.042714 0.447368 + 3 24 472 14 600 0.048387 0.631579 + 4 0 0 38 1072 NaN 0.000000 + 5 21 457 17 615 0.043933 0.552632 + 10 0 0 38 1072 NaN 0.000000 + 30 0 0 38 1072 NaN 0.000000 + 60 0 0 0 1110 NaN NaN + 90 0 0 0 1110 NaN NaN + 180 0 0 0 1110 NaN NaN + +Saved to datasets/flood_detection_evaluation.csv \ No newline at end of file diff --git a/src/models/__pycache__/climate_anomaly_model.cpython-311.pyc b/src/models/__pycache__/climate_anomaly_model.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c97dbcd6de01122e9e35a8ab8ce9fe9655b3183 Binary files /dev/null and b/src/models/__pycache__/climate_anomaly_model.cpython-311.pyc differ diff --git a/src/models/climate_anomaly_model.py b/src/models/climate_anomaly_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f5ff12d04ad3aebf799b998784978121baee80ed --- /dev/null +++ b/src/models/climate_anomaly_model.py @@ -0,0 +1,264 @@ +""" +Anomaly model for the climate sub-model: predicts (actual - climatology) +at a requested horizon, given a recent history of anomalies. Combined +with climate_climatology.py's climatology at inference time: + + forecast(date, horizon) = climatology(day_of_year) + predicted_anomaly(horizon) + +DESIGN CHOICE, stated directly rather than left implicit: the predicted +anomaly is passed through a learnable, per-variable, MONOTONICALLY +DECAYING gate as a function of horizon -- built in architecturally, not +left for training to discover on its own. This is a direct response to +what the real climatology results showed: precipitation has close to +zero seasonal predictability even at the baseline level (4/20 real grid +cells beat a flat mean, and the margins were negligible), so there is +no reason to expect a data-driven model to have genuine multi-month +signal for it either. Rather than risk the model outputting a +confident-looking but meaningless anomaly at day 180, the gate forces +the anomaly's contribution toward zero as horizon grows -- how fast is +LEARNED per variable (a temperature-like variable with real persistence +should decay slowly; precipitation should decay fast), but the +DIRECTION (toward zero, never away from it) is guaranteed by +construction, not hoped for. + +TESTING STATUS: prepare_training_windows and decay_gate are pure +numpy/pandas, tested directly below. The actual nn.Module +(ClimateAnomalyModel) requires torch, which is not installed in this +environment -- it has NOT been executed, only syntax-checked. Please +run scripts/test_climate_anomaly_model.py (uses this module) against +real data and report back what actually happens on the first run, +the same way the climatology and grid-extraction work was validated. +""" +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd + +try: + import torch + import torch.nn as nn + _HAS_TORCH = True +except ImportError: + _HAS_TORCH = False + nn = None # only referenced inside the class body below, never at import time otherwise + + +# --------------------------------------------------------------------------- +# Data preparation -- pure numpy/pandas, fully testable without torch +# --------------------------------------------------------------------------- + +def prepare_training_windows( + anomaly_wide: pd.DataFrame, lookback_days: int, horizons: List[int], +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[pd.Timestamp]]: + """ + Slices a [date x cell] anomaly DataFrame into (history, target) + training examples: for every date with a full lookback window + behind it AND real target values at every requested horizon ahead + of it, extract the lookback window as input and the horizon-ahead + anomalies as targets. + + A window is only kept if the ENTIRE lookback period is real + (non-NaN) for that cell -- a history with gaps silently interpolated + would teach the model on fabricated input, not real conditions. A + target is NaN-masked per (example, horizon) individually rather + than requiring every horizon to be real, since discharge/climate + series have real, uneven gaps and requiring all horizons real would + throw away most of the sparse real data this project already has. + + Args: + anomaly_wide: [date x cell_id], from climate_climatology. + compute_anomalies. + lookback_days: how much history feeds the encoder. + horizons: which day-ahead offsets to produce targets for, e.g. + [1, 5, 10, 30, 60, 90, 180]. + + Returns: + X: [n_examples, lookback_days, n_cells] -- history window, + same set of cells for every example (aligned columns). + Y: [n_examples, n_horizons] -- target anomaly at each horizon, + for the SPECIFIC cell/column indicated by `example_cell_idx`. + example_cell_idx: [n_examples] -- which column (cell) index in + X's last dimension this example's Y targets refer to (one + training example = one cell's history predicting that SAME + cell's own future, not cross-cell). + dates: the anchor date (the "today") for each example, for + traceability. + """ + max_horizon = max(horizons) + values = anomaly_wide.values + dates = anomaly_wide.index + n_dates, n_cells = values.shape + + X_list, Y_list, cell_idx_list, anchor_dates = [], [], [], [] + + for t in range(lookback_days, n_dates - max_horizon): + window = values[t - lookback_days:t, :] + window_complete_mask = ~np.isnan(window).any(axis=0) # per cell + if not window_complete_mask.any(): + continue + + for cell in np.where(window_complete_mask)[0]: + targets = np.array([values[t + h - 1, cell] for h in horizons]) + if np.isnan(targets).all(): + continue + X_list.append(window[:, cell]) + Y_list.append(targets) + cell_idx_list.append(cell) + anchor_dates.append(dates[t]) + + if not X_list: + return (np.empty((0, lookback_days)), np.empty((0, len(horizons))), + np.empty((0,), dtype=int), []) + + X = np.stack(X_list)[:, :, None] # [n_examples, lookback_days, 1] -- single-variable per call + Y = np.stack(Y_list) + return X, Y, np.array(cell_idx_list), anchor_dates + + +def decay_gate(horizons: np.ndarray, tau: float) -> np.ndarray: + """ + exp(-h / tau) -- the monotonic decay curve gating the predicted + anomaly toward zero as horizon grows. tau is learned per variable + in the real model (see ClimateAnomalyModel); this standalone + version is what's actually tested here, since the shape/behavior + of the curve is the real design claim, independent of whether it's + a fixed constant or a learned nn.Parameter wrapping this same formula. + + tau is the horizon (in days) at which the gate has decayed to 1/e + (~37%) of its starting value -- a small tau means fast decay + (appropriate for precipitation, per the real climatology results), + a large tau means slow decay (appropriate for temperature). + """ + return np.exp(-horizons / tau) + + +# --------------------------------------------------------------------------- +# Model -- requires torch, NOT executed in this environment (see module +# docstring). Syntax-checked only. +# --------------------------------------------------------------------------- + +if _HAS_TORCH: + class ClimateAnomalyModel(nn.Module): + """ + Encoder: GRU over the lookback window of anomalies -> final + hidden state per example. + Decoder: [hidden_state, horizon_embedding] -> MLP -> (mean, + log_variance) per horizon. + + Two complementary mechanisms handle "the model shouldn't be + confidently wrong when there's no real signal" -- confirmed as + a real, not hypothetical, problem: the first version of this + model (mean-only, plain MSE) was measurably WORSE than pure + climatology at several horizons for temp_C/solar_Wm2/ + wind_speed_ms, while a persistence baseline diagnostic showed + there was no real short-horizon signal for those variables to + find in the first place -- the model was outputting confident, + wrong guesses instead of admitting it didn't know. + + 1. The MEAN is passed through a learnable, monotonically + decaying gate per variable (see decay_gate) -- a HARD + architectural constraint: the model is structurally unable + to output a large mean anomaly at a horizon its own learned + tau says is past the point of real signal. Proven to work + correctly on real data already: trained on snow_mm (a + genuinely close-to-unpredictable variable), tau collapsed to + 2.3 and the model's predictions converged to matching + climatology almost exactly at every horizon beyond a few days. + 2. The VARIANCE is predicted separately, NOT gated the same + way, and trained via Gaussian NLL loss rather than plain + MSE. This is what actually targets the diagnosed + overconfidence problem: NLL rewards the model for outputting + high predicted variance when it genuinely can't predict + well, rather than being forced into a single point guess + the loss then punishes regardless of whether real signal + existed to make a better one. + + One model per climate variable, not a shared model across all + nine -- the real climatology and persistence-diagnostic results + showed genuinely different predictability profiles (runoff_mm + beat climatology at every horizon; wind_v_ms/precip_mm/snow_mm + essentially never did), and forcing one shared decay/variance + behavior to serve both would misrepresent at least one of them. + """ + + def __init__(self, hidden_dim: int = 32, horizon_embed_dim: int = 8, init_tau: float = 30.0): + super().__init__() + self.encoder = nn.GRU(input_size=1, hidden_size=hidden_dim, batch_first=True) + self.horizon_embed = nn.Linear(1, horizon_embed_dim) + self.decoder = nn.Sequential( + nn.Linear(hidden_dim + horizon_embed_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 2), # [mean, log_variance] per horizon + ) + # log_tau, not tau directly, so tau = exp(log_tau) is always + # positive regardless of what gradient descent does to the + # underlying parameter -- a negative or zero tau would make + # the decay gate nonsensical (blow up or divide by zero). + self.log_tau = nn.Parameter(torch.log(torch.tensor(float(init_tau)))) + + def forward(self, history: "torch.Tensor", horizons: "torch.Tensor") -> Tuple["torch.Tensor", "torch.Tensor"]: + """ + Args: + history: [batch, lookback_days, 1] -- real anomaly + history, already NaN-free (prepare_training_windows + only emits complete windows). + horizons: [n_horizons] -- day-ahead offsets to predict + for, shared across the batch. + + Returns: + (mean, log_variance), each [batch, n_horizons]. The mean + is already gated by the learned decay curve; log_variance + is the model's own learned, per-example uncertainty, + deliberately not put through the same gate -- a model + should be able to express high uncertainty at ANY + horizon, including short ones, if a specific example + genuinely warrants it, not only according to a fixed + schedule. + """ + _, h_n = self.encoder(history) + hidden = h_n.squeeze(0) # [batch, hidden_dim] + + batch_size = hidden.shape[0] + n_horizons = horizons.shape[0] + hidden_expanded = hidden.unsqueeze(1).expand(batch_size, n_horizons, hidden.shape[-1]) + horizon_feat = self.horizon_embed(horizons.float().view(1, n_horizons, 1).expand(batch_size, n_horizons, 1)) + + decoder_input = torch.cat([hidden_expanded, horizon_feat], dim=-1) + raw_output = self.decoder(decoder_input) # [batch, n_horizons, 2] + raw_mean = raw_output[..., 0] + log_var = raw_output[..., 1] + + tau = torch.exp(self.log_tau) + gate = torch.exp(-horizons.float() / tau) # [n_horizons] + mean = raw_mean * gate.unsqueeze(0) + + return mean, log_var + + + def gaussian_nll_loss(mean: "torch.Tensor", log_var: "torch.Tensor", target: "torch.Tensor", + min_log_var: float = -10.0) -> "torch.Tensor": + """ + Heteroscedastic Gaussian negative log-likelihood, NaN-masked + (same masking principle used throughout this project's physics + losses -- a target that doesn't exist for a given [example, + horizon] pair must not silently poison every other real pair's + gradient the same way an un-masked MSE mean would). + + NLL = 0.5 * log_var + 0.5 * (target - mean)^2 / exp(log_var) + + min_log_var clamps how confident the model is allowed to claim + to be -- without a floor, a model can trivially drive NLL toward + -infinity by predicting an extremely small variance on an + example it happens to get right by chance, which is a real, + well-known failure mode of unconstrained heteroscedastic + regression, not a hypothetical one. + """ + mask = ~torch.isnan(target) + if not mask.any(): + return mean.sum() * float("nan") + log_var_clamped = torch.clamp(log_var[mask], min=min_log_var) + residual_sq = (target[mask] - mean[mask]) ** 2 + return (0.5 * log_var_clamped + 0.5 * residual_sq / torch.exp(log_var_clamped)).mean() +else: + ClimateAnomalyModel = None # torch not installed; import guard, not a silent no-op + gaussian_nll_loss = None \ No newline at end of file