Upload folder using huggingface_hub (part 2)

.gitattributes

@@ -226,3 +226,9 @@ datasets/evaluation/station_timeseries_H420021012_h1.png filter=lfs diff=lfs mer
 datasets/evaluation/station_timeseries_H605641101_h1.png filter=lfs diff=lfs merge=lfs -text
 datasets/evaluation/station_timeseries_H605641201_h1.png filter=lfs diff=lfs merge=lfs -text
 datasets/evaluation/station_timeseries_H605641401_h1.png filter=lfs diff=lfs merge=lfs -text
+datasets/spatiotemporal_gnn/cnn/selected_subgraph.png filter=lfs diff=lfs merge=lfs -text
+datasets/spatiotemporal_gnn/cnn/training_curves.png filter=lfs diff=lfs merge=lfs -text
+datasets/spatiotemporal_gnn/gru/selected_subgraph.png filter=lfs diff=lfs merge=lfs -text
+datasets/spatiotemporal_gnn/lstm/selected_subgraph.png filter=lfs diff=lfs merge=lfs -text
+datasets/spatiotemporal_gnn/lstm/training_curves.png filter=lfs diff=lfs merge=lfs -text
+scripts/evaluation/__pycache__/evaluate_flood_detection.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text

datasets/spatiotemporal_gnn/archive/model_20260916_230515.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aefbe28c1a48651f1773b0515175c49b9d98206c2e9151daba890638d067c29b
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260917_045239.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ce381b9a9c4367df807e4c97a546517e646a3770046d035469fd964cf8cc02a
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260917_061838.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b202f12046c22c586fb9147fcb3b268e676a57d635601445290463c3079c7987
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260917_150452.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94f750c2f01878db314152abbd73efa710284fe2a2696876b2c478543609e3c9
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260917_232412.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1445a59ebcf57c85fcf355d069fb20ba87dcc3f50cb8cba0b69efbc6d3c179a
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260918_111742.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a84de0eb7b83e12f27ad27d3f853cebc0a5da4c28b04d31857e1785dec1b01e
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260918_141053.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:135216ca94621a2c139d322d60c01b72aec6cc70bce0fe2b1c71db774586a8a2
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260918_181508.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9d965937473c6a53193e92c65520d48d2bcc247c99f554c1375ef63c5298102
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260919_152545.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3f083196600d57d2bb047106f193ee4cca4c738f540bf6258eaa1797f62f670
+size 223022

datasets/spatiotemporal_gnn/archive/model_20260920_082410.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b74ebe9100364b358a0feeb49749becf925fcc194cf79a4785d5e616b5e2998
+size 223022

datasets/spatiotemporal_gnn/cnn/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:341a9946c45601c85f8959aee8c6656e33475c4c4c26f553a012d1bfe7635d4b
+size 976650

datasets/spatiotemporal_gnn/cnn/logs.txt

@@ -0,0 +1,89 @@
+epoch 1/100: train loss = 0.2286, val Q loss = 0.0920, val H loss = 0.0663, val combined = 0.1584 (best)
+  saved real best checkpoint (epoch 1, val_loss=0.1584) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=92.2145, len(train_loader)=562
+epoch 2/100: train loss = 0.1641, val Q loss = 0.0478, val H loss = 0.0481, val combined = 0.0958 (best)
+  saved real best checkpoint (epoch 2, val_loss=0.0958) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=81.9266, len(train_loader)=562
+epoch 3/100: train loss = 0.1458, val Q loss = 0.0433, val H loss = 0.0471, val combined = 0.0904 (best)
+  saved real best checkpoint (epoch 3, val_loss=0.0904) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=76.7376, len(train_loader)=562
+epoch 4/100: train loss = 0.1365, val Q loss = 0.0373, val H loss = 0.0398, val combined = 0.0771 (best)
+  saved real best checkpoint (epoch 4, val_loss=0.0771) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=74.5746, len(train_loader)=562
+epoch 5/100: train loss = 0.1327, val Q loss = 0.0527, val H loss = 0.0468, val combined = 0.0995
+  [debug] raw accounting: n_batches=562, total_loss=72.7644, len(train_loader)=562
+epoch 6/100: train loss = 0.1295, val Q loss = 0.0404, val H loss = 0.0400, val combined = 0.0804
+  [debug] raw accounting: n_batches=562, total_loss=71.7557, len(train_loader)=562
+epoch 7/100: train loss = 0.1277, val Q loss = 0.0422, val H loss = 0.0432, val combined = 0.0854
+  [debug] raw accounting: n_batches=562, total_loss=70.6966, len(train_loader)=562
+epoch 8/100: train loss = 0.1258, val Q loss = 0.0424, val H loss = 0.0399, val combined = 0.0823
+  [debug] raw accounting: n_batches=562, total_loss=70.5859, len(train_loader)=562
+epoch 9/100: train loss = 0.1256, val Q loss = 0.0362, val H loss = 0.0393, val combined = 0.0755 (best)
+  saved real best checkpoint (epoch 9, val_loss=0.0755) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=70.4025, len(train_loader)=562
+epoch 10/100: train loss = 0.1253, val Q loss = 0.0472, val H loss = 0.0420, val combined = 0.0892
+  [debug] raw accounting: n_batches=562, total_loss=69.8427, len(train_loader)=562
+epoch 11/100: train loss = 0.1243, val Q loss = 0.0462, val H loss = 0.0410, val combined = 0.0872
+  [debug] raw accounting: n_batches=562, total_loss=69.2181, len(train_loader)=562
+epoch 12/100: train loss = 0.1232, val Q loss = 0.0441, val H loss = 0.0470, val combined = 0.0911
+  [debug] raw accounting: n_batches=562, total_loss=68.5817, len(train_loader)=562
+epoch 13/100: train loss = 0.1220, val Q loss = 0.0476, val H loss = 0.0404, val combined = 0.0880
+  [debug] raw accounting: n_batches=562, total_loss=68.7422, len(train_loader)=562
+epoch 14/100: train loss = 0.1223, val Q loss = 0.0411, val H loss = 0.0375, val combined = 0.0785
+  [debug] raw accounting: n_batches=562, total_loss=68.5472, len(train_loader)=562
+epoch 15/100: train loss = 0.1220, val Q loss = 0.0415, val H loss = 0.0397, val combined = 0.0812
+  [debug] raw accounting: n_batches=562, total_loss=68.0892, len(train_loader)=562
+epoch 16/100: train loss = 0.1212, val Q loss = 0.0400, val H loss = 0.0363, val combined = 0.0763
+  [debug] raw accounting: n_batches=562, total_loss=67.2499, len(train_loader)=562
+epoch 17/100: train loss = 0.1197, val Q loss = 0.0419, val H loss = 0.0393, val combined = 0.0812
+  [debug] raw accounting: n_batches=562, total_loss=67.1504, len(train_loader)=562
+epoch 18/100: train loss = 0.1195, val Q loss = 0.0373, val H loss = 0.0377, val combined = 0.0749 (best)
+  saved real best checkpoint (epoch 18, val_loss=0.0749) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=67.1872, len(train_loader)=562
+epoch 19/100: train loss = 0.1196, val Q loss = 0.0407, val H loss = 0.0414, val combined = 0.0821
+  [debug] raw accounting: n_batches=562, total_loss=67.6086, len(train_loader)=562
+epoch 20/100: train loss = 0.1203, val Q loss = 0.0381, val H loss = 0.0419, val combined = 0.0800
+  [debug] raw accounting: n_batches=562, total_loss=66.7986, len(train_loader)=562
+epoch 21/100: train loss = 0.1189, val Q loss = 0.0531, val H loss = 0.0470, val combined = 0.1001
+  [debug] raw accounting: n_batches=562, total_loss=66.389, len(train_loader)=562
+epoch 22/100: train loss = 0.1181, val Q loss = 0.0378, val H loss = 0.0347, val combined = 0.0725 (best)
+  saved real best checkpoint (epoch 22, val_loss=0.0725) to datasets/spatiotemporal_gnn/cnn/checkpoint.pt
+  [debug] raw accounting: n_batches=562, total_loss=66.3914, len(train_loader)=562
+epoch 23/100: train loss = 0.1181, val Q loss = 0.0518, val H loss = 0.0401, val combined = 0.0919
+  [debug] raw accounting: n_batches=562, total_loss=65.8369, len(train_loader)=562
+epoch 24/100: train loss = 0.1171, val Q loss = 0.0398, val H loss = 0.0377, val combined = 0.0775
+  [debug] raw accounting: n_batches=562, total_loss=65.9454, len(train_loader)=562
+epoch 25/100: train loss = 0.1173, val Q loss = 0.0406, val H loss = 0.0383, val combined = 0.0789
+  [debug] raw accounting: n_batches=562, total_loss=65.8238, len(train_loader)=562
+epoch 26/100: train loss = 0.1171, val Q loss = 0.0478, val H loss = 0.0458, val combined = 0.0936
+  [debug] raw accounting: n_batches=562, total_loss=65.5497, len(train_loader)=562
+epoch 27/100: train loss = 0.1166, val Q loss = 0.0399, val H loss = 0.0385, val combined = 0.0785
+  [debug] raw accounting: n_batches=562, total_loss=65.4061, len(train_loader)=562
+epoch 28/100: train loss = 0.1164, val Q loss = 0.0427, val H loss = 0.0373, val combined = 0.0800
+  [debug] raw accounting: n_batches=562, total_loss=65.1854, len(train_loader)=562
+epoch 29/100: train loss = 0.1160, val Q loss = 0.0461, val H loss = 0.0400, val combined = 0.0861
+  [debug] raw accounting: n_batches=562, total_loss=64.7025, len(train_loader)=562
+epoch 30/100: train loss = 0.1151, val Q loss = 0.0430, val H loss = 0.0394, val combined = 0.0825
+  [debug] raw accounting: n_batches=562, total_loss=64.3632, len(train_loader)=562
+epoch 31/100: train loss = 0.1145, val Q loss = 0.0382, val H loss = 0.0356, val combined = 0.0738
+  [debug] raw accounting: n_batches=562, total_loss=64.5384, len(train_loader)=562
+epoch 32/100: train loss = 0.1148, val Q loss = 0.0372, val H loss = 0.0355, val combined = 0.0728
+stopped early at epoch 32
+Saved final real model (bare state_dict, for evaluate_flood_detection.py etc.) to datasets/spatiotemporal_gnn/cnn/model.pt
+(The real resumable checkpoint at datasets/spatiotemporal_gnn/cnn/checkpoint.pt -- model+optimizer+epoch+history -- already held these exact same best real weights since epoch 22; this final save is a convenience copy in the plain format the evaluation scripts expect.)
+
+Best validation loss: 0.0725
+Learned Manning's roughness n = 0.1574 (0.03 was the initialization, real natural streams typically fall in ~0.025-0.1; now sigmoid-bounded to [0.01, 0.3] by construction, so it can no longer drift outside that range the way it did before -- but landing at or very near either boundary (0.01 or 0.3) still means it saturated rather than converged to a real interior value, and is still worth investigating, not just noting)
+Learned bank height = 0.10m (2.5m was the initialization, an assumption with no real measured data behind it -- compare this against real rating-curve data for the gauges with one, if the model's learned value looks implausible for these specific rivers)
+Saved model to datasets/spatiotemporal_gnn/cnn/model.pt
+
+Naive baseline, discharge (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1298
+Naive baseline, water level (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1239
+Discharge: model beats the naive baseline by 70.9% -- real, not just baseline-matching skill
+Water level: model beats the naive baseline by 72.0% -- real, not just baseline-matching skill
+
+python -m scripts.training.train_spatiotemporal_gnn \
+    --data-root datasets \
+    --train-end 2025-06-30 --val-end 2025-12-31 \
+    --log-space-discharge-loss --nse-weight 0 --water-balance-weight 0 \
+    --event-reweight --temporal-type cnn

datasets/spatiotemporal_gnn/cnn/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2820c042b1a741443bc6595021c4d2f5b76ce3357a96dd5d60bca32651ac694
+size 322010

datasets/spatiotemporal_gnn/gru/logs.txt

@@ -0,0 +1,51 @@
+epoch 1/100: train loss = 0.2174, val Q loss = 0.0584, val H loss = 0.0508, val combined = 0.1091 (best)
+  [debug] raw accounting: n_batches=562, total_loss=78.6866, len(train_loader)=562
+epoch 2/100: train loss = 0.1400, val Q loss = 0.0398, val H loss = 0.0456, val combined = 0.0853 (best)
+  [debug] raw accounting: n_batches=562, total_loss=75.0667, len(train_loader)=562
+epoch 3/100: train loss = 0.1336, val Q loss = 0.0539, val H loss = 0.0490, val combined = 0.1029
+  [debug] raw accounting: n_batches=562, total_loss=72.6642, len(train_loader)=562
+epoch 4/100: train loss = 0.1293, val Q loss = 0.0416, val H loss = 0.0480, val combined = 0.0896
+  [debug] raw accounting: n_batches=562, total_loss=71.4249, len(train_loader)=562
+epoch 5/100: train loss = 0.1271, val Q loss = 0.0384, val H loss = 0.0389, val combined = 0.0773 (best)
+  [debug] raw accounting: n_batches=562, total_loss=69.9505, len(train_loader)=562
+epoch 6/100: train loss = 0.1245, val Q loss = 0.0347, val H loss = 0.0421, val combined = 0.0768 (best)
+  [debug] raw accounting: n_batches=562, total_loss=69.2856, len(train_loader)=562
+epoch 7/100: train loss = 0.1233, val Q loss = 0.0376, val H loss = 0.0430, val combined = 0.0806
+  [debug] raw accounting: n_batches=562, total_loss=68.9054, len(train_loader)=562
+epoch 8/100: train loss = 0.1226, val Q loss = 0.0378, val H loss = 0.0428, val combined = 0.0806
+  [debug] raw accounting: n_batches=562, total_loss=67.9286, len(train_loader)=562
+epoch 9/100: train loss = 0.1209, val Q loss = 0.0405, val H loss = 0.0400, val combined = 0.0805
+  [debug] raw accounting: n_batches=562, total_loss=67.6105, len(train_loader)=562
+epoch 10/100: train loss = 0.1203, val Q loss = 0.0343, val H loss = 0.0369, val combined = 0.0711 (best)
+  [debug] raw accounting: n_batches=562, total_loss=67.4021, len(train_loader)=562
+epoch 11/100: train loss = 0.1199, val Q loss = 0.0355, val H loss = 0.0364, val combined = 0.0719
+  [debug] raw accounting: n_batches=562, total_loss=67.1944, len(train_loader)=562
+epoch 12/100: train loss = 0.1196, val Q loss = 0.0451, val H loss = 0.0403, val combined = 0.0855
+  [debug] raw accounting: n_batches=562, total_loss=66.9262, len(train_loader)=562
+epoch 13/100: train loss = 0.1191, val Q loss = 0.0358, val H loss = 0.0387, val combined = 0.0746
+  [debug] raw accounting: n_batches=562, total_loss=66.9341, len(train_loader)=562
+epoch 14/100: train loss = 0.1191, val Q loss = 0.0395, val H loss = 0.0451, val combined = 0.0846
+  [debug] raw accounting: n_batches=562, total_loss=66.25, len(train_loader)=562
+epoch 15/100: train loss = 0.1179, val Q loss = 0.0374, val H loss = 0.0422, val combined = 0.0796
+  [debug] raw accounting: n_batches=562, total_loss=65.8047, len(train_loader)=562
+epoch 16/100: train loss = 0.1171, val Q loss = 0.0347, val H loss = 0.0370, val combined = 0.0717
+  [debug] raw accounting: n_batches=562, total_loss=65.8151, len(train_loader)=562
+epoch 17/100: train loss = 0.1171, val Q loss = 0.0430, val H loss = 0.0439, val combined = 0.0870
+  [debug] raw accounting: n_batches=562, total_loss=65.825, len(train_loader)=562
+epoch 18/100: train loss = 0.1171, val Q loss = 0.0352, val H loss = 0.0387, val combined = 0.0738
+  [debug] raw accounting: n_batches=562, total_loss=65.1963, len(train_loader)=562
+epoch 19/100: train loss = 0.1160, val Q loss = 0.0393, val H loss = 0.0394, val combined = 0.0787
+  [debug] raw accounting: n_batches=562, total_loss=65.3031, len(train_loader)=562
+epoch 20/100: train loss = 0.1162, val Q loss = 0.0381, val H loss = 0.0407, val combined = 0.0788
+stopped early at epoch 20
+Archived previous checkpoint to datasets/spatiotemporal_gnn/archive/model_20260920_082410.pt
+
+Best validation loss: 0.0711
+Learned Manning's roughness n = 0.1566 (0.03 was the initialization, real natural streams typically fall in ~0.025-0.1; now sigmoid-bounded to [0.01, 0.3] by construction, so it can no longer drift outside that range the way it did before -- but landing at or very near either boundary (0.01 or 0.3) still means it saturated rather than converged to a real interior value, and is still worth investigating, not just noting)
+Learned bank height = 0.10m (2.5m was the initialization, an assumption with no real measured data behind it -- compare this against real rating-curve data for the gauges with one, if the model's learned value looks implausible for these specific rivers)
+Saved model to datasets/spatiotemporal_gnn/model.pt
+
+Naive baseline, discharge (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1298
+Naive baseline, water level (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1239
+Discharge: model beats the naive baseline by 73.6% -- real, not just baseline-matching skill
+Water level: model beats the naive baseline by 70.2% -- real, not just baseline-matching skill

datasets/spatiotemporal_gnn/gru/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5e4425095867eb29f471bf1f8e26837e106085e38ae0423b8b5991664c5f53
+size 223022

datasets/spatiotemporal_gnn/lstm/logs.txt

@@ -0,0 +1,50 @@
+epoch 1/100: train loss = 0.2043, val Q loss = 0.0629, val H loss = 0.0504, val combined = 0.1133 (best)
+  [debug] raw accounting: n_batches=562, total_loss=81.4646, len(train_loader)=562
+epoch 2/100: train loss = 0.1450, val Q loss = 0.0450, val H loss = 0.0408, val combined = 0.0858 (best)
+  [debug] raw accounting: n_batches=562, total_loss=76.7782, len(train_loader)=562
+epoch 3/100: train loss = 0.1366, val Q loss = 0.0678, val H loss = 0.0544, val combined = 0.1222
+  [debug] raw accounting: n_batches=562, total_loss=73.7316, len(train_loader)=562
+epoch 4/100: train loss = 0.1312, val Q loss = 0.0410, val H loss = 0.0371, val combined = 0.0781 (best)
+  [debug] raw accounting: n_batches=562, total_loss=71.938, len(train_loader)=562
+epoch 5/100: train loss = 0.1280, val Q loss = 0.0672, val H loss = 0.0421, val combined = 0.1093
+  [debug] raw accounting: n_batches=562, total_loss=70.4825, len(train_loader)=562
+epoch 6/100: train loss = 0.1254, val Q loss = 0.0424, val H loss = 0.0446, val combined = 0.0871
+  [debug] raw accounting: n_batches=562, total_loss=69.2004, len(train_loader)=562
+epoch 7/100: train loss = 0.1231, val Q loss = 0.0370, val H loss = 0.0382, val combined = 0.0752 (best)
+  [debug] raw accounting: n_batches=562, total_loss=68.1995, len(train_loader)=562
+epoch 8/100: train loss = 0.1214, val Q loss = 0.0450, val H loss = 0.0431, val combined = 0.0882
+  [debug] raw accounting: n_batches=562, total_loss=68.2369, len(train_loader)=562
+epoch 9/100: train loss = 0.1214, val Q loss = 0.0394, val H loss = 0.0332, val combined = 0.0726 (best)
+  [debug] raw accounting: n_batches=562, total_loss=67.716, len(train_loader)=562
+epoch 10/100: train loss = 0.1205, val Q loss = 0.0345, val H loss = 0.0351, val combined = 0.0696 (best)
+  [debug] raw accounting: n_batches=562, total_loss=67.628, len(train_loader)=562
+epoch 11/100: train loss = 0.1203, val Q loss = 0.0489, val H loss = 0.0428, val combined = 0.0918
+  [debug] raw accounting: n_batches=562, total_loss=66.7925, len(train_loader)=562
+epoch 12/100: train loss = 0.1188, val Q loss = 0.0365, val H loss = 0.0343, val combined = 0.0708
+  [debug] raw accounting: n_batches=562, total_loss=66.7536, len(train_loader)=562
+epoch 13/100: train loss = 0.1188, val Q loss = 0.0439, val H loss = 0.0429, val combined = 0.0868
+  [debug] raw accounting: n_batches=562, total_loss=65.9348, len(train_loader)=562
+epoch 14/100: train loss = 0.1173, val Q loss = 0.0369, val H loss = 0.0366, val combined = 0.0734
+  [debug] raw accounting: n_batches=562, total_loss=66.2568, len(train_loader)=562
+epoch 15/100: train loss = 0.1179, val Q loss = 0.0414, val H loss = 0.0416, val combined = 0.0830
+  [debug] raw accounting: n_batches=562, total_loss=65.7003, len(train_loader)=562
+epoch 16/100: train loss = 0.1169, val Q loss = 0.0366, val H loss = 0.0365, val combined = 0.0731
+  [debug] raw accounting: n_batches=562, total_loss=65.4524, len(train_loader)=562
+epoch 17/100: train loss = 0.1165, val Q loss = 0.0353, val H loss = 0.0361, val combined = 0.0714
+  [debug] raw accounting: n_batches=562, total_loss=64.6876, len(train_loader)=562
+epoch 18/100: train loss = 0.1151, val Q loss = 0.0357, val H loss = 0.0373, val combined = 0.0730
+  [debug] raw accounting: n_batches=562, total_loss=64.794, len(train_loader)=562
+epoch 19/100: train loss = 0.1153, val Q loss = 0.0454, val H loss = 0.0451, val combined = 0.0904
+  [debug] raw accounting: n_batches=562, total_loss=64.9283, len(train_loader)=562
+epoch 20/100: train loss = 0.1155, val Q loss = 0.0380, val H loss = 0.0408, val combined = 0.0788
+stopped early at epoch 20
+
+Best validation loss: 0.0696
+Learned Manning's roughness n = 0.1566 (0.03 was the initialization, real natural streams typically fall in ~0.025-0.1; now sigmoid-bounded to [0.01, 0.3] by construction, so it can no longer drift outside that range the way it did before -- but landing at or very near either boundary (0.01 or 0.3) still means it saturated rather than converged to a real interior value, and is still worth investigating, not just noting)
+Learned bank height = 0.10m (2.5m was the initialization, an assumption with no real measured data behind it -- compare this against real rating-curve data for the gauges with one, if the model's learned value looks implausible for these specific rivers)
+Saved model to datasets/spatiotemporal_gnn/lstm/model.pt
+
+Naive baseline, discharge (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1298
+Naive baseline, water level (real training-period empirical quantiles [0.5, 0.9, 0.95, 0.99]): 0.1239
+Discharge: model beats the naive baseline by 73.4% -- real, not just baseline-matching skill
+Water level: model beats the naive baseline by 71.7% -- real, not just baseline-matching skill

datasets/spatiotemporal_gnn/lstm/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3fa27cdd263f5dcb1b7a140a24b01621344283703c0fe091717d043b96a4924
+size 256366

datasets/spatiotemporal_gnn/model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:debcf20e5f2fa6309d3fccacb0b4631bbe6a319625698a6a03c06d195b1d3022
+oid sha256:3c5e4425095867eb29f471bf1f8e26837e106085e38ae0423b8b5991664c5f53
 size 223022

requirements.txt

@@ -44,3 +44,5 @@ pytest-cov>=4.1.0
 # GIS processing tools (optional but useful)
 # GDAL binaries should be installed separately
 # pyproj>=3.5.0
+#0761197809 - Celine
+#0722645224 - Shem 

scripts/analysis/check_discharge_quantile_distribution.py

@@ -0,0 +1,104 @@
+"""
+Computes the real empirical quantile distribution of discharge for
+every real gauged station -- built to answer a real, specific question
+before deciding how to re-weight or restructure the model's predicted
+quantiles: where do real near-zero/low-flow, normal-flow, and peak
+events actually fall for each real station, rather than assuming the
+current quantile set [0.5, 0.9, 0.95, 0.99] is well-matched to the
+real data's actual shape.
+
+Reuses build_discharge_timeseries directly (the same real, established
+loading path train_spatiotemporal_gnn.py itself uses) and the same
+real per-basin filtering fix check_dry_valleys.py needed -- 
+build_discharge_timeseries returns every real gauged station across
+BOTH basins regardless of which nodes_df is passed in, confirmed
+directly in that earlier diagnostic.
+
+Usage:
+    python -m scripts.analysis.check_discharge_quantile_distribution --data-root datasets
+"""
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+QUANTILE_LEVELS = [0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Check the real empirical discharge quantile distribution per station")
+    parser.add_argument("--data-root", type=Path, default=Path("datasets"))
+    parser.add_argument("--start-date", type=str, default="2000-01-01",
+                         help="As wide a real historical window as available.")
+    parser.add_argument("--end-date", type=str, default="2026-12-31")
+    args = parser.parse_args()
+
+    try:
+        from src.graph.dynamic_features import build_discharge_timeseries
+    except ImportError:
+        sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+        from src.graph.dynamic_features import build_discharge_timeseries
+
+    date_range = (args.start_date, args.end_date)
+    rows = []
+
+    for basin in ["eure", "risle"]:
+        nodes_path = args.data_root / "reach_graph" / f"{basin}_nodes_enriched.csv"
+        if not nodes_path.exists():
+            print(f"{basin}: no real nodes file at {nodes_path}, skipping.")
+            continue
+        nodes_df = pd.read_csv(nodes_path)
+
+        wide = build_discharge_timeseries(nodes_df, args.data_root / "hydrometric", date_range)
+        # Real per-basin filtering, same fix check_dry_valleys.py needed:
+        # build_discharge_timeseries returns every real gauged station
+        # across BOTH basins, not filtered by the nodes_df passed in.
+        this_basin_codes = set(nodes_df["station_code"]) & set(wide.columns)
+        wide = wide[sorted(this_basin_codes)]
+
+        for station_code in wide.columns:
+            series = wide[station_code].dropna()
+            n_real = len(series)
+            if n_real == 0:
+                continue
+            real_quantiles = series.quantile(QUANTILE_LEVELS)
+            row = {"basin": basin, "station_code": station_code, "n_real": n_real}
+            for q_level, q_value in zip(QUANTILE_LEVELS, real_quantiles):
+                row[f"Q{int(q_level*100):02d}"] = float(q_value)
+            rows.append(row)
+
+    if not rows:
+        print("No real stations found in either basin -- nothing to report.")
+        return
+
+    result_df = pd.DataFrame(rows)
+    print("=" * 100)
+    print("Real empirical discharge quantile distribution, per real gauged station (m3/s)")
+    print("=" * 100)
+    print(result_df.to_string(index=False))
+
+    print()
+    print("=" * 100)
+    print("Real skew/spread ratios -- how far the tails sit from the real median, per station")
+    print("=" * 100)
+    for _, row in result_df.iterrows():
+        q01, q50, q99 = row["Q01"], row["Q50"], row["Q99"]
+        low_ratio = q50 / q01 if q01 > 0 else float("inf")
+        high_ratio = q99 / q50 if q50 > 0 else float("inf")
+        print(f"{row['station_code']} ({row['basin']}): Q50/Q01={low_ratio:>8.1f}x, "
+              f"Q99/Q50={high_ratio:>8.1f}x  (real Q01={q01:.2f}, Q50={q50:.2f}, Q99={q99:.2f})")
+
+    print()
+    print("=" * 100)
+    print("Where does this project's CURRENT model quantile set [0.5, 0.9, 0.95, 0.99] actually land?")
+    print("=" * 100)
+    print("No real LOW-tail quantile exists in the current model output at all -- Q50 is the lowest "
+          "quantile predicted, meaning there is no dedicated, explicit low-flow signal in the model's "
+          "own output structure, only whatever Q50 happens to learn implicitly.")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/check_dry_valleys.py

@@ -0,0 +1,115 @@
+"""
+Checks whether any real gauged station in this project's actual data
+ever shows true dry-valley (zero or near-zero) discharge -- a direct,
+factual data check, built to resolve whether zero-inflation-specific
+loss functions (Tweedie, dual-head zero-inflated models, etc.) are
+actually relevant here, rather than assuming either way from generic
+hydrology advice that wasn't written for this specific river system.
+
+Reuses build_discharge_timeseries directly (the same real, established
+loading path train_spatiotemporal_gnn.py itself uses), not a
+re-derived reading of the raw hydrometric files -- avoids the exact
+mistake the first version of check_routing_lag_distribution.py made by
+not reusing the real, existing function.
+
+Usage:
+    python -m scripts.analysis.check_dry_valleys --data-root datasets
+"""
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Check for real dry-valley (zero/near-zero discharge) conditions")
+    parser.add_argument("--data-root", type=Path, default=Path("datasets"))
+    parser.add_argument("--start-date", type=str, default="2000-01-01",
+                         help="As wide a real historical window as available -- a dry-valley check "
+                              "benefits from more real history, not just the training window.")
+    parser.add_argument("--end-date", type=str, default="2026-12-31")
+    parser.add_argument("--near-zero-threshold-m3s", type=float, default=0.01,
+                         help="A real, small discharge (L/s equivalent: 10 L/s) below which flow is "
+                              "treated as 'near-zero' for reporting purposes -- exact zero is checked "
+                              "separately and explicitly, this threshold is only for the 'near' case.")
+    args = parser.parse_args()
+
+    try:
+        from src.graph.dynamic_features import build_discharge_timeseries
+    except ImportError:
+        sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+        from src.graph.dynamic_features import build_discharge_timeseries
+
+    date_range = (args.start_date, args.end_date)
+    rows = []
+
+    for basin in ["eure", "risle"]:
+        nodes_path = args.data_root / "reach_graph" / f"{basin}_nodes_enriched.csv"
+        if not nodes_path.exists():
+            print(f"{basin}: no real nodes file at {nodes_path}, skipping.")
+            continue
+        nodes_df = pd.read_csv(nodes_path)
+
+        wide = build_discharge_timeseries(nodes_df, args.data_root / "hydrometric", date_range)
+        # build_discharge_timeseries returns EVERY real gauged station
+        # across BOTH basins (confirmed directly -- it never filters by
+        # the nodes_df passed in; that filtering happens later, inside
+        # assemble_dynamic_tensor, which this script doesn't call) --
+        # restricting to this basin's own real station_codes here is
+        # required, or every station gets reported twice, once per
+        # basin, with identical values. A real, caught mistake in an
+        # earlier version of this exact script.
+        this_basin_codes = set(nodes_df["station_code"]) & set(wide.columns)
+        wide = wide[sorted(this_basin_codes)]
+        print(f"{basin}: real discharge data for {wide.shape[1]} real gauged station(s), "
+              f"{wide.shape[0]} real date(s) in range")
+
+        for station_code in wide.columns:
+            series = wide[station_code].dropna()
+            n_real = len(series)
+            if n_real == 0:
+                rows.append({"basin": basin, "station_code": station_code, "n_real": 0,
+                             "min_m3s": np.nan, "pct_exact_zero": np.nan,
+                             "pct_near_zero": np.nan, "p01_m3s": np.nan, "p05_m3s": np.nan})
+                continue
+            n_zero = int((series == 0).sum())
+            n_near_zero = int((series <= args.near_zero_threshold_m3s).sum())
+            rows.append({
+                "basin": basin, "station_code": station_code, "n_real": n_real,
+                "min_m3s": float(series.min()),
+                "pct_exact_zero": 100.0 * n_zero / n_real,
+                "pct_near_zero": 100.0 * n_near_zero / n_real,
+                "p01_m3s": float(series.quantile(0.01)),
+                "p05_m3s": float(series.quantile(0.05)),
+            })
+
+    if not rows:
+        print("No real stations found in either basin -- nothing to report.")
+        return
+
+    result_df = pd.DataFrame(rows).sort_values("min_m3s")
+    print()
+    print("=" * 70)
+    print(f"Real per-station discharge floor (near-zero threshold: {args.near_zero_threshold_m3s} m3/s)")
+    print("=" * 70)
+    print(result_df.to_string(index=False))
+
+    n_ever_exact_zero = int((result_df["pct_exact_zero"] > 0).sum())
+    n_ever_near_zero = int((result_df["pct_near_zero"] > 0).sum())
+    print()
+    if n_ever_exact_zero == 0:
+        print("No real station in this dataset ever recorded an exact-zero discharge day -- "
+              "zero-inflation-specific losses (Tweedie, dual-head zero-inflated models) do not "
+              "appear to match this project's actual real data, based on real evidence.")
+    else:
+        print(f"{n_ever_exact_zero} real station(s) recorded at least one exact-zero discharge day -- "
+              f"real, direct evidence a zero-inflation-aware approach could be worth investigating "
+              f"further for those specific stations.")
+    print(f"{n_ever_near_zero} real station(s) recorded at least one real day at or below "
+          f"{args.near_zero_threshold_m3s} m3/s.")
+
+
+if __name__ == "__main__":
+    main()

scripts/analysis/check_routing_lag_distribution.py

@@ -0,0 +1,112 @@
+"""
+Computes the real travel-time lag for every real edge in the ACTUAL
+collapsed training subgraph (the ~150-edge, 100-node graph
+train_spatiotemporal_gnn.py really trains on), not the full,
+fine-grained reach graph.
+
+CORRECTED VERSION: an earlier version of this script read the raw
+reach_graph/{basin}_edges.csv files directly -- real, but the WRONG
+edges. Those are the full, uncollapsed tronçon-level graph (~3300-5200
+edges per basin), where most real distances between adjacent nodes are
+tiny (a few hundred meters), giving travel times of minutes to hours.
+The graph actually used for training is the COLLAPSED subgraph
+(build_collapsed_subgraph), where chains through dropped virtual nodes
+are merged into single, real, much longer edges (summed distance_km/
+elevation_drop_m) -- confirmed as the real, meaningful distinction: a
+first run against the wrong edges showed 100% of edges at zero real
+lag days, which would have been a real, actionable, and WRONG
+conclusion about this project's actual routing physics.
+
+Usage:
+    python -m scripts.analysis.check_routing_lag_distribution --data-root datasets
+"""
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+
+def estimate_travel_time_hours(distance_km, elevation_drop_m, min_velocity_ms=0.1, max_velocity_ms=3.0):
+    """Exact real formula from src/graph/physics_losses.py's own
+    estimate_travel_time_hours -- kept in sync manually since this is a
+    read-only diagnostic script, not meant to import the training
+    pipeline's own torch-dependent modules."""
+    distance_m = distance_km * 1000.0
+    slope = np.clip(elevation_drop_m / np.maximum(distance_m, 1.0), 1e-6, None)
+    velocity = np.clip(np.sqrt(slope) * 10.0, min_velocity_ms, max_velocity_ms)
+    return distance_m / velocity / 3600.0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Check the real routing lag distribution on the actual collapsed training subgraph")
+    parser.add_argument("--data-root", type=Path, default=Path("datasets"))
+    parser.add_argument("--max-nodes", type=int, default=100,
+                         help="Must match train_spatiotemporal_gnn.py's own --max-nodes for this to "
+                              "reflect the real, actual training graph.")
+    parser.add_argument("--timestep-hours", type=float, default=24.0)
+    args = parser.parse_args()
+
+    try:
+        from src.graph.subgraph_selection import build_collapsed_subgraph
+    except ImportError:
+        sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+        from src.graph.subgraph_selection import build_collapsed_subgraph
+
+    max_nodes_per_basin = args.max_nodes // 2
+    all_lags_days = []
+
+    for basin in ["eure", "risle"]:
+        nodes_path = args.data_root / "reach_graph" / f"{basin}_nodes_enriched.csv"
+        edges_path = args.data_root / "reach_graph" / f"{basin}_edges.csv"
+        if not nodes_path.exists() or not edges_path.exists():
+            print(f"{basin}: missing {nodes_path} or {edges_path}, skipping.")
+            continue
+
+        full_nodes_df = pd.read_csv(nodes_path)
+        full_edges_df = pd.read_csv(edges_path)
+        print(f"{basin}: {len(full_nodes_df)} real node(s), {len(full_edges_df)} real edge(s) "
+              f"(full reach graph)")
+
+        small_nodes_df, small_edges_df = build_collapsed_subgraph(
+            full_nodes_df, full_edges_df, max_nodes=max_nodes_per_basin,
+        )
+        n_edges = len(small_edges_df)
+        print(f"{basin}: collapsed to {len(small_nodes_df)} real node(s), {n_edges} real "
+              f"collapsed edge(s) -- THIS is the real graph training actually uses")
+
+        if n_edges == 0:
+            print(f"{basin}: zero real collapsed edges -- nothing to report.")
+            continue
+
+        drop = small_edges_df["elevation_drop_m"].fillna(0.1).clip(lower=0.1)
+        hours = estimate_travel_time_hours(small_edges_df["distance_km"].values, drop.values)
+        days = hours / 24.0
+        lag_steps = np.round(hours / args.timestep_hours).astype(int)
+
+        print(f"  Real collapsed-edge distance (km): min={small_edges_df['distance_km'].min():.2f}, "
+              f"median={small_edges_df['distance_km'].median():.2f}, max={small_edges_df['distance_km'].max():.2f}")
+        print(f"  Real travel time (days): min={days.min():.3f}, median={np.median(days):.3f}, "
+              f"mean={days.mean():.3f}, max={days.max():.3f}, p90={np.percentile(days, 90):.3f}, "
+              f"p99={np.percentile(days, 99):.3f}")
+        print(f"  Real lag (whole {args.timestep_hours:.0f}h timesteps): "
+              f"min={lag_steps.min()}, median={int(np.median(lag_steps))}, max={lag_steps.max()}")
+        all_lags_days.extend(days.tolist())
+
+    if not all_lags_days:
+        print("No real collapsed edges found in either basin -- nothing to report.")
+        return
+
+    all_lags_days = np.array(all_lags_days)
+    print()
+    print("=" * 70)
+    print("Real combined lag distribution, both basins (real collapsed training graph)")
+    print("=" * 70)
+    for count in [1, 2, 3, 5, 7, 10, 14, 21]:
+        coverage = float((all_lags_days <= count).mean() * 100)
+        print(f"ROUTING_HORIZON_COUNT={count:>2}: covers {coverage:.1f}% of real collapsed edges' own real travel time")
+
+
+if __name__ == "__main__":
+    main()

scripts/evaluation/logs.txt

@@ -0,0 +1,9 @@
+python -m scripts.evaluation.evaluate_flood_detection \
+  --train-end 2025-06-30 \
+  --val-end 2025-12-31 \
+  --plot-stations H404021101,H420021012 \
+  --blend-quantiles \
+  --blend-quantiles-max-horizon 5 \
+  --compare-all-quantiles \
+  --trace-fp-inputs 5 \
+  --dump-fp-details

scripts/training/__pycache__/train_spatiotemporal_gnn.cpython-311.pyc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a54bb608b5e1626b3243cf3c627f452242ffb421b66816bd29b3e9bfe84fcd87
-size 106555
+oid sha256:a660ca66432e728a84b0c4fb9e408937a04476c458b247f26c715874c1946e1e
+size 130424

scripts/training/train_spatiotemporal_gnn.py

@@ -26,37 +26,49 @@ from torch.utils.data import Dataset, DataLoader
 
 try:
     from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN, masked_mse_graph
-    from src.graph.pinball_loss import multi_quantile_loss
+    from src.graph.pinball_loss import (
+        multi_quantile_loss, log_space_multi_quantile_loss, per_station_relative_multi_quantile_loss,
+        weighted_multi_quantile_loss,
+    )
     from src.graph.dynamic_features import (
         build_discharge_timeseries, build_waterlevel_timeseries, build_groundwater_timeseries,
         build_precipitation_timeseries, assemble_dynamic_tensor, compute_rolling_precip_sum,
         fit_specific_discharge_coefficient, estimate_discharge_from_precip,
         build_forecast_tensor, FORECAST_LEAD_TIMES, compute_days_since_last_real, compress_days_since_last_real,
+        compute_forward_filled_tensor,
     )
     from src.graph.physics_losses import (
         build_confluence_index, build_braid_index, build_routing_index,
         confluence_mass_balance_loss, split_rejoin_conservation_loss, routing_consistency_loss,
         manning_consistency_loss, water_balance_loss, routing_soft_dtw_loss,
-        nse_loss, compute_per_node_historical_mean,
+        nse_loss, compute_per_node_historical_mean, kge_loss_from_pooled_pairs,
+        compute_per_node_historical_std, compute_per_node_historical_median,
     )
+    from src.graph.flood_events import build_event_lookup, build_event_weight_tensor
     from src.graph.subgraph_selection import build_collapsed_subgraph
 except ImportError:
     import sys
     sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
     from src.graph.spatiotemporal_gnn import prepare_graph_training_windows, SpatiotemporalGNN, masked_mse_graph
-    from src.graph.pinball_loss import multi_quantile_loss
+    from src.graph.pinball_loss import (
+        multi_quantile_loss, log_space_multi_quantile_loss, per_station_relative_multi_quantile_loss,
+        weighted_multi_quantile_loss,
+    )
     from src.graph.dynamic_features import (
         build_discharge_timeseries, build_waterlevel_timeseries, build_groundwater_timeseries,
         build_precipitation_timeseries, assemble_dynamic_tensor, compute_rolling_precip_sum,
         fit_specific_discharge_coefficient, estimate_discharge_from_precip,
         build_forecast_tensor, FORECAST_LEAD_TIMES, compute_days_since_last_real, compress_days_since_last_real,
+        compute_forward_filled_tensor,
     )
     from src.graph.physics_losses import (
         build_confluence_index, build_braid_index, build_routing_index,
         confluence_mass_balance_loss, split_rejoin_conservation_loss, routing_consistency_loss,
         manning_consistency_loss, water_balance_loss, routing_soft_dtw_loss,
-        nse_loss, compute_per_node_historical_mean,
+        nse_loss, compute_per_node_historical_mean, kge_loss_from_pooled_pairs,
+        compute_per_node_historical_std, compute_per_node_historical_median,
     )
+    from src.graph.flood_events import build_event_lookup, build_event_weight_tensor
     from src.graph.subgraph_selection import build_collapsed_subgraph
 
 BASIN_FILE_NAMES = {0: "eure", 1: "risle"}
@@ -80,14 +92,15 @@ QUANTILES = [0.5, 0.9, 0.95, 0.99]
 # deliberate ceiling matching this project's planned future real
 # Météo-France forecast horizon, not an arbitrary round number.
 HORIZONS = [1, 2, 3, 4, 5, 7, 10, 14, 21]
-ROUTING_HORIZON_COUNT = 7  # first 7 days -- extended from 5 to match the real max ECMWF
-# forecast lead time (7, now in HORIZONS) -- routing_consistency_loss and Soft-DTW compare
-# the model's own predictions against each other, so extending them carries no real new
-# risk. water_balance_loss is the one term this extension is more borderline for (it
-# checks predicted discharge against HISTORICAL, already-observed precipitation, which
-# genuinely stops explaining discharge the further out the prediction horizon goes) -- 7
-# is still within a real, defensible range; this constant should not be pushed further
-# without separately reconsidering water_balance_loss's own real physical justification.
+ROUTING_HORIZON_COUNT = 3  # real, measured lag distribution on the ACTUAL collapsed
+# training subgraph (check_routing_lag_distribution.py) showed 100% real edge coverage
+# already at 3 days, with the longest real edge (Eure, 38.87km) rounding to a 2-day lag
+# -- 7 was based on an incorrect assumption (checked against the raw, uncollapsed reach
+# graph, where distances are misleadingly tiny) and provided zero real routing benefit,
+# while needlessly extending water_balance_loss's reach to days where its own real
+# physical justification (recent rainfall explaining discharge) is weaker -- a real,
+# measured precision regression at every short horizon followed that change. 3 is the
+# real, evidence-based minimum with a genuine one-day margin, not a boundary-hugging value.
 
 
 def get_dynamic_channel_index(var_names: List[str], target_var: str, channels_per_var: int = 2) -> Tuple[int, int]:
@@ -788,6 +801,7 @@ class GraphWindowDataset(Dataset):
     def __init__(
         self, X: np.ndarray, Y: np.ndarray, Y_mask: np.ndarray, Y_H: np.ndarray, Y_H_mask: np.ndarray,
         forecast_precip: np.ndarray, forecast_missing: np.ndarray,
+        event_weight: Optional[np.ndarray] = None,
     ):
         self.X = torch.tensor(X, dtype=torch.float32)
         self.Y = torch.tensor(Y, dtype=torch.float32)
@@ -796,13 +810,21 @@ class GraphWindowDataset(Dataset):
         self.Y_H_mask = torch.tensor(Y_H_mask, dtype=torch.bool)
         self.forecast_precip = torch.tensor(forecast_precip, dtype=torch.float32)
         self.forecast_missing = torch.tensor(forecast_missing, dtype=torch.float32)
+        # --event-reweight's per-(example, node, horizon) discharge loss
+        # weight -- defaults to an all-1.0 array (an exact no-op, see
+        # main()'s own comment on why this is safe) when the caller
+        # doesn't have real event data to build a real one from.
+        self.event_weight = torch.tensor(
+            event_weight if event_weight is not None else np.ones_like(Y, dtype=np.float32),
+            dtype=torch.float32,
+        )
 
     def __len__(self) -> int:
         return self.X.shape[0]
 
     def __getitem__(self, idx: int):
         return (self.X[idx], self.Y[idx], self.Y_mask[idx], self.Y_H[idx], self.Y_H_mask[idx],
-                self.forecast_precip[idx], self.forecast_missing[idx])
+                self.forecast_precip[idx], self.forecast_missing[idx], self.event_weight[idx])
 
 
 def plot_selected_subgraph(basin_data: List[Tuple[pd.DataFrame, pd.DataFrame, int]], combined: Dict, output_path: Path) -> None:
@@ -942,7 +964,7 @@ def compute_naive_baseline_loss(
     """
     naive_quantiles_t = torch.tensor(naive_quantiles, dtype=torch.float32)
     total, n = 0.0, 0
-    for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, _, _ in val_loader:
+    for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, _, _, _ in val_loader:
         target_batch = Y_batch if target == "Q" else Y_H_batch
         for i in range(X_batch.shape[0]):
             Y_i = target_batch[i]
@@ -969,6 +991,16 @@ def train_spatiotemporal_model(
     soft_dtw_gamma: float = 1.0,
     discharge_per_node_mean: Optional["torch.Tensor"] = None,
     waterlevel_per_node_mean: Optional["torch.Tensor"] = None,
+    use_log_space_discharge_loss: bool = False,
+    log_discharge_offset: Optional[float] = None,
+    use_per_station_relative_loss: bool = False,
+    discharge_per_node_std: Optional["torch.Tensor"] = None,
+    use_kge_loss: bool = False,
+    kge_min_points: int = 5,
+    checkpoint_path: Optional[Path] = None,
+    resume_from: Optional[Path] = None,
+    waterlevel_mean: float = 0.0,
+    waterlevel_std: float = 1.0,
 ) -> Dict:
     """
     One real graph shared across every example in a batch -- looping
@@ -1013,15 +1045,54 @@ def train_spatiotemporal_model(
     # wait for completion or early stopping.
     history = {"train_loss": [], "val_Q_loss": [], "val_H_loss": [], "val_combined": [], "best_epoch": None}
 
+    # --resume: restores real model/optimizer state, the epoch counter,
+    # the real best-loss-so-far, and real loss history from a real
+    # checkpoint saved by an earlier (possibly interrupted) run of THIS
+    # function, so training continues from the next real epoch instead
+    # of starting over from freshly initialized weights. Silently starts
+    # fresh if resume_from is None or the real file doesn't exist yet --
+    # this is expected on a normal first run, not an error.
+    start_epoch = 0
+    if resume_from is not None and Path(resume_from).exists():
+        checkpoint = torch.load(resume_from, map_location=device)
+        model.load_state_dict(checkpoint["model_state"])
+        optimizer.load_state_dict(checkpoint["optimizer_state"])
+        start_epoch = checkpoint["epoch"] + 1
+        best_val_loss = checkpoint["best_val_loss"]
+        best_val_Q_loss = checkpoint["best_val_Q_loss"]
+        best_val_H_loss = checkpoint["best_val_H_loss"]
+        epochs_without_improvement = checkpoint.get("epochs_without_improvement", 0)
+        history = checkpoint.get("history", history)
+        best_state = {k: v.clone() for k, v in model.state_dict().items()}
+        print(f"Resumed from real checkpoint {resume_from}: continuing at epoch {start_epoch+1}, "
+              f"real best_val_loss so far = {best_val_loss:.4f}, "
+              f"epochs_without_improvement = {epochs_without_improvement}")
+
     debug_examples_remaining = 25  # widened from 3 -- the first 3 examples all looked fine, need more coverage to catch whatever's different about the examples that actually break
     skipped_batches_to_report = 10  # report the actual batch_loss value for the first several skipped batches, not just that it happened
 
-    for epoch in range(max_epochs):
+    for epoch in range(start_epoch, max_epochs):
         model.train()
         total_loss, n_batches = 0.0, 0
-        for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch in train_loader:
+        for (X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch,
+             event_weight_batch) in train_loader:
             optimizer.zero_grad()
             batch_loss = X_batch.sum() * 0.0  # zero, correct dtype/device
+            # Real, per-node pooling for KGE's correlation term -- unlike
+            # every other loss in this loop, real KGE needs multiple
+            # real, PAIRED (observed, predicted) points for the SAME
+            # node to mean anything (a correlation from one example's
+            # sparse ground truth alone is not real evidence of timing
+            # skill). Pooled across every real example in this BATCH,
+            # per node, then KGE is computed once per node AFTER the
+            # example loop completes -- a genuinely different
+            # accumulation pattern from every other per-example loss
+            # term here, not a bug.
+            n_nodes_batch = x_shared_static.shape[0]
+            pooled_obs_Q_by_node = [[] for _ in range(n_nodes_batch)]
+            pooled_pred_Q_by_node = [[] for _ in range(n_nodes_batch)]
+            pooled_obs_H_by_node = [[] for _ in range(n_nodes_batch)]
+            pooled_pred_H_by_node = [[] for _ in range(n_nodes_batch)]
             for i in range(X_batch.shape[0]):
                 x_dynamic_seq = X_batch[i].to(device)
                 Y_i = Y_batch[i].to(device)
@@ -1030,6 +1101,7 @@ def train_spatiotemporal_model(
                 Y_H_mask_i = Y_H_mask_batch[i].to(device)
                 forecast_precip_i = forecast_precip_batch[i].to(device)
                 forecast_missing_i = forecast_missing_batch[i].to(device)
+                event_weight_i = event_weight_batch[i].to(device)
 
                 pred = model(x_shared_static, x_gauge_static, x_dynamic_seq, is_gauged, edge_index, edge_attr, basin_id, horizons_t,
                              forecast_precip_i, forecast_missing_i)
@@ -1038,6 +1110,45 @@ def train_spatiotemporal_model(
                 Q_pred_quantiles = pred[:, :, 0, :]
                 H_pred_quantiles = pred[:, :, 1, :]
 
+                if use_kge_loss:
+                    # Gather this example's real (non-NaN) (observed,
+                    # predicted-median) pairs into the batch-level pool,
+                    # per node -- real ground truth is sparse per
+                    # example, so a real, meaningful correlation for a
+                    # given node only emerges once several real anchor
+                    # dates' worth of pairs accumulate across the batch.
+                    #
+                    # CRITICAL, CONFIRMED FIX: un-standardize to REAL,
+                    # physical units (real L/s, real cm) HERE, before
+                    # pooling -- confirmed as a real, structural bug,
+                    # not a rare edge case: standardized (z-scored) data
+                    # is deliberately centered at mean~0, which makes
+                    # KGE's bias-ratio term (pred_mean/obs_mean) blow up
+                    # whenever a small, sparse pooled sample's obs_mean
+                    # lands near zero, which is EXPECTED to happen
+                    # regularly for standardized data, not rarely.
+                    # Measured directly: worst-case loss 8,260 on
+                    # standardized-space-like pooled samples, only 4.07
+                    # on real, physical-unit values (see
+                    # kge_loss_from_pooled_pairs' own docstring).
+                    Q_median_i = Q_pred_quantiles[:, :, 0]
+                    H_median_i = H_pred_quantiles[:, :, 0]
+                    Y_i_real = Y_i * discharge_std + discharge_mean
+                    Q_median_i_real = Q_median_i * discharge_std + discharge_mean
+                    Y_H_i_real = Y_H_i * waterlevel_std + waterlevel_mean
+                    H_median_i_real = H_median_i * waterlevel_std + waterlevel_mean
+                    real_mask_Q = ~torch.isnan(Y_i)
+                    real_mask_H = ~torch.isnan(Y_H_i)
+                    for node_idx in range(n_nodes_batch):
+                        node_mask_q = real_mask_Q[node_idx]
+                        if node_mask_q.any():
+                            pooled_obs_Q_by_node[node_idx].append(Y_i_real[node_idx][node_mask_q])
+                            pooled_pred_Q_by_node[node_idx].append(Q_median_i_real[node_idx][node_mask_q])
+                        node_mask_h = real_mask_H[node_idx]
+                        if node_mask_h.any():
+                            pooled_obs_H_by_node[node_idx].append(Y_H_i_real[node_idx][node_mask_h])
+                            pooled_pred_H_by_node[node_idx].append(H_median_i_real[node_idx][node_mask_h])
+
                 # Pinball loss across every predicted quantile -- Y_i/Y_H_i
                 # are 2D [n_nodes, n_horizons] real targets, broadcast
                 # against each quantile prediction internally by
@@ -1047,7 +1158,48 @@ def train_spatiotemporal_model(
                 # from the same real target tensor), so pinball_loss's own
                 # NaN-masking already does the right thing without a
                 # separate mask argument.
-                supervised_Q = multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES)
+                # event_weight_i (see --event-reweight) is ALWAYS passed
+                # into the discharge supervised term below -- an all-1.0
+                # tensor when the flag is off, which reproduces each
+                # branch's original unweighted behavior exactly (see
+                # main()'s own comment on why this is a safe no-op).
+                # Deliberately NOT mutually exclusive with log-space or
+                # per-station-relative -- event-reweighting addresses a
+                # different axis (how many gradient updates a flood-
+                # adjacent example gets), not value scale, so it composes
+                # with either of the two scale-focused losses below.
+                if use_log_space_discharge_loss:
+                    # Scoped to discharge only -- real, confirmed
+                    # peak-specific bias (25.0% PBIAS at high flow vs.
+                    # -1.8% at low flow, real station evaluation) is a
+                    # discharge-specific finding; water level doesn't
+                    # share discharge's orders-of-magnitude real skew,
+                    # so it stays on the original, linear-space loss.
+                    supervised_Q = log_space_multi_quantile_loss(
+                        Y_i, Q_pred_quantiles, QUANTILES, discharge_mean, discharge_std,
+                        log_offset=log_discharge_offset, sample_weight=event_weight_i,
+                    )
+                elif use_per_station_relative_loss:
+                    # A genuinely different dimension of the same real
+                    # scale problem log-space addresses: log-space
+                    # compresses the gap between a real low-flow error
+                    # and a real peak error WITHIN one station; this
+                    # compresses the gap between a real error at a
+                    # small station and a real error at a large
+                    # station, ACROSS stations -- real, direct evidence
+                    # (check_discharge_quantile_distribution.py) showed
+                    # this project's 6 real gauges span real Q50 values
+                    # from ~868 to ~9323 L/s. Mutually exclusive with
+                    # log-space by design here, not stacked -- the same
+                    # real lesson from log-space+KGE both independently
+                    # causing the same regression applies: isolate one
+                    # new mechanism at a time before combining any.
+                    supervised_Q = per_station_relative_multi_quantile_loss(
+                        Y_i, Q_pred_quantiles, QUANTILES, discharge_mean, discharge_std,
+                        discharge_per_node_std, sample_weight=event_weight_i,
+                    )
+                else:
+                    supervised_Q = weighted_multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES, event_weight_i)
                 supervised_H = multi_quantile_loss(Y_H_i, H_pred_quantiles, QUANTILES)
 
                 # NSE: a real, different signal from pinball loss above --
@@ -1168,6 +1320,65 @@ def train_spatiotemporal_model(
                 batch_loss = batch_loss + example_loss
 
             batch_loss = batch_loss / X_batch.shape[0]
+
+            if use_kge_loss:
+                # A real, BATCH-level term -- added AFTER the per-example
+                # division above, not before: kge_loss_total is computed
+                # once per node from pooled real pairs across the whole
+                # batch, not once per example, so it must not get divided
+                # by the per-example count the way example_loss terms are.
+                kge_loss_total = X_batch.sum() * 0.0
+                n_nodes_with_kge = 0
+                for node_idx in range(n_nodes_batch):
+                    if pooled_obs_Q_by_node[node_idx]:
+                        obs_cat = torch.cat(pooled_obs_Q_by_node[node_idx])
+                        pred_cat = torch.cat(pooled_pred_Q_by_node[node_idx])
+                        if len(obs_cat) >= kge_min_points:
+                            kge_loss_total = kge_loss_total + kge_loss_from_pooled_pairs(obs_cat, pred_cat)
+                            n_nodes_with_kge += 1
+                    if pooled_obs_H_by_node[node_idx]:
+                        obs_cat_h = torch.cat(pooled_obs_H_by_node[node_idx])
+                        pred_cat_h = torch.cat(pooled_pred_H_by_node[node_idx])
+                        if len(obs_cat_h) >= kge_min_points:
+                            kge_loss_total = kge_loss_total + kge_loss_from_pooled_pairs(obs_cat_h, pred_cat_h)
+                            n_nodes_with_kge += 1
+                if n_nodes_with_kge > 0:
+                    kge_contribution = physics_weights.get("kge", 1.0) * (kge_loss_total / n_nodes_with_kge)
+                    batch_loss = batch_loss + kge_contribution
+                    if debug_examples_remaining > 0:
+                        # Real, BATCH-level debug visibility -- this term
+                        # doesn't fit the existing per-example debug
+                        # block at all (it isn't computed until after
+                        # that whole loop finishes), so it needs its own
+                        # print, not a line inside the per-example block.
+                        # A real, previously-caught gap: NSE got this
+                        # same fix after being wired in without it; KGE
+                        # needs the same discipline from the start.
+                        #
+                        # requires_grad, not a per-tensor autograd.grad
+                        # check, is the real, correct thing to verify
+                        # here: kge_contribution depends on POOLED
+                        # tensors from potentially several different
+                        # examples' Q_pred_quantiles/H_pred_quantiles,
+                        # not just the last example's from the loop --
+                        # checking gradient against that one specific,
+                        # likely-uninvolved tensor would silently report
+                        # "finite" via allow_unused=True even if the
+                        # real gradient path were broken. The finite-
+                        # gradient check across every model parameter
+                        # together already happens later, at
+                        # clip_grad_norm_, after backward() -- this
+                        # print confirms the term is even part of the
+                        # real, differentiable graph at all.
+                        print(f"    [debug] kge_loss_total: {kge_loss_total.item():.6g}, "
+                              f"n_nodes_with_kge={n_nodes_with_kge}/{n_nodes_batch}, "
+                              f"weighted_contribution={kge_contribution.item():.6g}, "
+                              f"finite={torch.isfinite(kge_contribution).item()}, "
+                              f"requires_grad={kge_contribution.requires_grad}")
+                elif debug_examples_remaining > 0:
+                    print(f"    [debug] kge_loss_total: NO node cleared kge_min_points={kge_min_points} "
+                          f"this batch -- KGE contributed nothing this step.")
+
             if not torch.isfinite(batch_loss):
                 if skipped_batches_to_report > 0:
                     skipped_batches_to_report -= 1
@@ -1217,7 +1428,8 @@ def train_spatiotemporal_model(
         val_Q_total, val_Q_batches = 0.0, 0
         val_H_total, val_H_batches = 0.0, 0
         with torch.no_grad():
-            for X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch in val_loader:
+            for (X_batch, Y_batch, mask_batch, Y_H_batch, Y_H_mask_batch, forecast_precip_batch, forecast_missing_batch,
+                 event_weight_batch) in val_loader:
                 for i in range(X_batch.shape[0]):
                     x_dynamic_seq = X_batch[i].to(device)
                     Y_i = Y_batch[i].to(device)
@@ -1226,12 +1438,30 @@ def train_spatiotemporal_model(
                     Y_H_mask_i = Y_H_mask_batch[i].to(device)
                     forecast_precip_i = forecast_precip_batch[i].to(device)
                     forecast_missing_i = forecast_missing_batch[i].to(device)
+                    event_weight_i = event_weight_batch[i].to(device)
 
                     pred = model(x_shared_static, x_gauge_static, x_dynamic_seq, is_gauged, edge_index, edge_attr, basin_id, horizons_t,
                                  forecast_precip_i, forecast_missing_i)
                     Q_pred_quantiles = pred[:, :, 0, :]
                     H_pred_quantiles = pred[:, :, 1, :]
-                    supervised_Q = multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES)
+                    # Same event-reweighted loss branch as the train loop
+                    # (kept identical on purpose) -- val loss should measure
+                    # the model against the SAME objective it was optimized
+                    # for, so early stopping/model selection stays consistent
+                    # with what training loss actually minimized, rather than
+                    # silently reverting to the unweighted loss here.
+                    if use_log_space_discharge_loss:
+                        supervised_Q = log_space_multi_quantile_loss(
+                            Y_i, Q_pred_quantiles, QUANTILES, discharge_mean, discharge_std,
+                            log_offset=log_discharge_offset, sample_weight=event_weight_i,
+                        )
+                    elif use_per_station_relative_loss:
+                        supervised_Q = per_station_relative_multi_quantile_loss(
+                            Y_i, Q_pred_quantiles, QUANTILES, discharge_mean, discharge_std,
+                            discharge_per_node_std, sample_weight=event_weight_i,
+                        )
+                    else:
+                        supervised_Q = weighted_multi_quantile_loss(Y_i, Q_pred_quantiles, QUANTILES, event_weight_i)
                     supervised_H = multi_quantile_loss(Y_H_i, H_pred_quantiles, QUANTILES)
                     # isfinite, not isnan -- confirmed as a real bug against
                     # real training: an untrained model produced an
@@ -1275,6 +1505,35 @@ def train_spatiotemporal_model(
             best_state = {k: v.clone() for k, v in model.state_dict().items()}
             epochs_without_improvement = 0
             history["best_epoch"] = epoch + 1
+            # Real, immediate on-disk save of the best weights -- not
+            # just held in best_state until this whole function returns.
+            # A real crash/interruption any time after this line still
+            # leaves the best real weights (plus optimizer state/epoch/
+            # history, for --resume) safely on disk from THIS epoch,
+            # rather than losing everything back to the last full
+            # training run's own final save.
+            if checkpoint_path is not None:
+                try:
+                    checkpoint_path.parent.mkdir(parents=True, exist_ok=True)
+                    tmp_checkpoint_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".tmp")
+                    torch.save({
+                        "model_state": best_state,
+                        "optimizer_state": optimizer.state_dict(),
+                        "epoch": epoch,
+                        "best_val_loss": best_val_loss,
+                        "best_val_Q_loss": best_val_Q_loss,
+                        "best_val_H_loss": best_val_H_loss,
+                        "epochs_without_improvement": epochs_without_improvement,
+                        "history": history,
+                    }, tmp_checkpoint_path)
+                    # Atomic rename -- so a real crash mid-write never
+                    # leaves a real half-written, unloadable checkpoint
+                    # file at checkpoint_path itself.
+                    tmp_checkpoint_path.replace(checkpoint_path)
+                    print(f"  saved real best checkpoint (epoch {epoch+1}, val_loss={val_loss:.4f}) "
+                          f"to {checkpoint_path}")
+                except Exception as e:
+                    print(f"  [warning] failed to save real checkpoint this epoch: {e}")
         else:
             epochs_without_improvement += 1
 
@@ -1291,7 +1550,8 @@ def train_spatiotemporal_model(
     if best_state is not None:
         model.load_state_dict(best_state)
     return {"best_val_loss": best_val_loss, "best_val_Q_loss": best_val_Q_loss,
-            "best_val_H_loss": best_val_H_loss, "model_state": model.state_dict()}
+            "best_val_H_loss": best_val_H_loss, "model_state": model.state_dict(),
+            "best_epoch": history.get("best_epoch")}
 
 
 def main() -> None:
@@ -1353,6 +1613,126 @@ def main() -> None:
                               "Applied to the median prediction only, same convention as every other "
                               "physics-adjacent loss in this file. Real per-node historical mean is "
                               "fit on the training period only, same discipline as discharge_mean/std.")
+    parser.add_argument("--log-space-discharge-loss", action="store_true",
+                         help="Opt-in, not yet the default -- computes the discharge supervised loss "
+                              "(pinball loss) in log(real L/s + log_discharge_offset) space instead of "
+                              "standardized space. Built after real evaluation confirmed a peak-specific "
+                              "bias (25.0% PBIAS at high flow vs. -1.8% at low flow, one real station). "
+                              "REAL, CONFIRMED CAVEAT: the original offset=1.0 (log1p) default weighted "
+                              "a low-flow error ~28x more than an identical high-flow error at real "
+                              "station scale, and a real training run with it produced a real, severe "
+                              "regression -- low-flow NSE collapsed to -116.7, PBIAS to -102.7%, at the "
+                              "same station this was built to help. See --log-discharge-offset, whose "
+                              "new default softens this. Mutually exclusive with "
+                              "--per-station-relative-loss by design -- pick one to test at a time. "
+                              "Scoped to discharge only -- water level doesn't share discharge's orders-"
+                              "of-magnitude real skew. Real validation loss (val_Q_loss) stays in the "
+                              "original, standardized space regardless of this flag, so it remains "
+                              "directly comparable across every run in this project, with or without "
+                              "this flag set -- only the TRAINING objective changes.")
+    parser.add_argument("--log-discharge-offset", type=float, default=None,
+                         help="The real 'c' in log(real L/s + c), for --log-space-discharge-loss. None "
+                              "(the default) resolves to 0.25*discharge_std at run time -- a real, "
+                              "measured, data-derived value, not an arbitrary guess: measured directly "
+                              "to weight a real low-flow error ~10x more than an identical real high-"
+                              "flow error (down from ~28x at the original, riskier offset=1.0/log1p). "
+                              "Pass 1.0 explicitly to reproduce the original log1p behavior (not "
+                              "recommended given the real regression it caused) -- larger values soften "
+                              "low-flow sensitivity further, at the real cost of weakening the exact "
+                              "peak-bias fix this loss exists for.")
+    parser.add_argument("--per-station-relative-loss", action="store_true",
+                         help="Opt-in -- normalizes each real node's real discharge by THAT NODE's own "
+                              "real historical std (see compute_per_node_historical_std) before computing "
+                              "the discharge supervised loss, instead of the single, GLOBAL discharge_std "
+                              "every other version of this loss uses. Built after real, direct evidence "
+                              "(check_discharge_quantile_distribution.py) showed this project's 6 real "
+                              "gauges span wildly different real scale (real Q50 from ~868 to ~9323 L/s) "
+                              "and skew (real Q50/Q01 from 1.4x to literally infinite) -- under global "
+                              "standardization, a real, proportionally severe error at a small station "
+                              "contributes much less loss than the identical real proportional error at "
+                              "a large station. Addresses a genuinely different dimension of the real "
+                              "scale problem than --log-space-discharge-loss does (across stations, not "
+                              "within one station's own low-vs-high flow) -- mutually exclusive with it "
+                              "by design, not stacked, so results can be attributed to one real change "
+                              "at a time. Scoped to discharge only, same reasoning as log-space.")
+    parser.add_argument("--use-kge-loss", action="store_true",
+                         help="Opt-in real, TEMPORAL Kling-Gupta Efficiency loss, meant to replace NSE "
+                              "(pass --nse-weight 0 alongside this for a clean A/B test), not stack with "
+                              "it. Unlike nse_loss, which only needs a node's own pre-computed historical "
+                              "mean and works fine per-example, real KGE's correlation term needs "
+                              "multiple real, PAIRED (observed, predicted) points for the SAME node to "
+                              "mean anything -- ground truth is too sparse within one example for that "
+                              "(20-50 real values spread across ~100 nodes on one real anchor date). This "
+                              "pools real pairs PER NODE ACROSS THE WHOLE BATCH before computing KGE once "
+                              "per node with enough real points (see --kge-min-points), a genuinely "
+                              "different, batch-level accumulation pattern from every other per-example "
+                              "loss in this file, not a bug. Applied to both discharge and water level "
+                              "medians, same scope as NSE had.")
+    parser.add_argument("--kge-min-points", type=int, default=5,
+                         help="Minimum real, pooled (observed, predicted) points required for a node "
+                              "before computing its real KGE contribution this batch -- a correlation "
+                              "from fewer real points isn't real evidence of timing skill (2 points give "
+                              "a correlation of exactly +-1 or an undefined value). Nodes below this "
+                              "threshold in a given batch simply don't contribute to KGE loss that step, "
+                              "not treated as zero loss.")
+    parser.add_argument("--kge-weight", type=float, default=0.3,
+                         help="Weight for --use-kge-loss. Defaults to 0.3, not 1.0 -- matching the real, "
+                              "measured lesson from NSE: full weight (1.0) caused a real, severe day-30 "
+                              "flood-detection collapse (tp 21->1), only partially recovered at 0.3. "
+                              "Since KGE is a new, less-tested loss with a genuinely different "
+                              "accumulation pattern (batch-pooled, not per-example), starting cautious "
+                              "rather than assuming full weight is safe.")
+    parser.add_argument("--event-reweight", action="store_true",
+                         help="Opt-in -- upweights the discharge supervised (pinball) loss for real "
+                              "(node, horizon) examples whose real TARGET date falls within "
+                              "--event-reweight-window-days of a real labeled flood event (from "
+                              "--events-path), boosted by --event-reweight-boost; every other example "
+                              "keeps the neutral weight of 1.0. Built as the direct response to a real, "
+                              "measured problem: --check-quantile-calibration on a real checkpoint showed "
+                              "opposite-direction miscalibration (q0.95 coverage 0.990 on low-flow days, "
+                              "0.698 on high-flow days -- both should be ~0.95) consistent with a model "
+                              "whose learned quantile function is dominated by the sheer VOLUME of quiet-"
+                              "day examples, since real flood events are rare (125 across the whole "
+                              "2013-2025 record) -- pinball loss's own per-sample asymmetry (a flood-day "
+                              "underprediction already costs more than a same-size quiet-day "
+                              "overprediction) doesn't fix a per-example-COUNT imbalance, which this "
+                              "targets directly instead. Composes with --log-space-discharge-loss AND "
+                              "with --per-station-relative-loss (deliberately NOT mutually exclusive with "
+                              "either -- this addresses a genuinely different axis, sample count, not "
+                              "value scale). Scoped to discharge only, same reasoning as log-space and "
+                              "per-station-relative. Requires real events at --events-path; if none are "
+                              "found, this flag is a no-op (weight stays 1.0 everywhere) and a warning is "
+                              "printed rather than failing the run.")
+    parser.add_argument("--events-path", type=Path, default=None,
+                         help="Defaults to <data-root>/flood_events_discharge.csv, same real file "
+                              "evaluate_flood_detection.py already reads -- only loaded when "
+                              "--event-reweight is set.")
+    parser.add_argument("--event-reweight-window-days", type=int, default=5,
+                         help="How many real days from a labeled event's start/end the boost still "
+                              "applies. Default 5 -- identify_flood_events.py's own docstring notes real "
+                              "event durations already run 3-10 days, so this catches the real rising/"
+                              "falling limb immediately around a labeled event, not just the event's own "
+                              "days.")
+    parser.add_argument("--event-reweight-boost", type=float, default=4.0,
+                         help="The real weight multiplier applied to boosted examples (an untouched "
+                              "example keeps weight 1.0). Default 4.0 -- a real, moderate value in the "
+                              "3-5x range; large enough to meaningfully shift gradient share toward "
+                              "flood-adjacent examples without risking the kind of real, severe "
+                              "regression this project has already hit from being too aggressive with a "
+                              "new loss mechanism (see --kge-weight's own docstring for a real precedent). "
+                              "Compare against a run with --event-reweight omitted (an exact no-op "
+                              "baseline, not just boost=1.0) before trusting this helps.")
+    parser.add_argument("--temporal-type", type=str, default="gru", choices=["gru", "lstm", "cnn"],
+                         help="A real, direct ablation switch for the per-node temporal encoder -- "
+                              "everything else (graph layers, physics losses, decoder) stays identical "
+                              "regardless of this choice, so comparing runs across the three isolates "
+                              "just this one component. 'gru' is the original, default architecture. "
+                              "'lstm' swaps in nn.LSTM (same interface, uses only its hidden state, not "
+                              "its cell state, downstream). 'cnn' uses a real, causal, dilated 1D conv "
+                              "stack (_CausalTemporalCNN in spatiotemporal_gnn.py) instead of a "
+                              "recurrent mechanism -- its causality (no future timestep ever leaking "
+                              "into an earlier output) was verified directly, not just reasoned about, "
+                              "before this option was added.")
     parser.add_argument("--batch-size", type=int, default=8,
                          help="Small default -- each example is a full graph forward pass looped "
                               "internally (see train_spatiotemporal_model's docstring), so this is "
@@ -1361,8 +1741,35 @@ def main() -> None:
     parser.add_argument("--patience", type=int, default=10)
     parser.add_argument("--lr", type=float, default=1e-3)
     parser.add_argument("--weight-decay", type=float, default=1e-4)
+    parser.add_argument("--checkpoint-path", type=Path, default=None,
+                         help="Where to save a real, RESUMABLE checkpoint (model weights + optimizer "
+                              "state + epoch counter + best-val-loss-so-far + real loss history) every "
+                              "single time validation loss improves during training -- not just the bare "
+                              "final model.pt, which is only ever written once the whole real training "
+                              "run completes. This makes an interrupted/crashed real run recoverable: the "
+                              "best real weights seen so far are always already on disk, updated the same "
+                              "epoch they're found, not held only in memory until training finishes. "
+                              "Defaults to <output_dir>/checkpoint.pt, i.e. the same real per-temporal_type "
+                              "directory model.pt itself already uses (datasets/spatiotemporal_gnn/ for "
+                              "gru, .../lstm/ or .../cnn/ otherwise).")
+    parser.add_argument("--resume", action="store_true",
+                         help="Resume training from --checkpoint-path (or its default location) if a "
+                              "real checkpoint exists there -- restores real model weights, optimizer "
+                              "state, the epoch counter, the best validation loss seen so far, "
+                              "epochs-without-improvement (for early stopping), and real loss history, "
+                              "then continues training from the next real epoch instead of starting over "
+                              "from a freshly initialized model. Silently starts fresh (same as omitting "
+                              "this flag) if no checkpoint is found at that real path yet.")
     args = parser.parse_args()
 
+    if args.log_space_discharge_loss and args.per_station_relative_loss:
+        print("--log-space-discharge-loss and --per-station-relative-loss are mutually exclusive by "
+              "design -- both address a real scale problem, but along different dimensions (within one "
+              "station vs. across stations), and this project's own real evidence (log-space + KGE both "
+              "independently causing the same real regression) argues for isolating one new mechanism "
+              "at a time before ever combining them. Pick one.")
+        return
+
     shared_static_cols = ["elevation_m", "catchment_area_km2", "idpr_value",
                           "avg_groundwater_level_m", "distance_to_nearest_cavity_km", "n_cavities_within_20km"]
     # Dropped raw latitude/longitude -- real risk of the model memorizing
@@ -1394,7 +1801,16 @@ def main() -> None:
     combined = combine_basins(basin_data, shared_static_cols, gauge_static_cols)
     print(f"Combined graph: {combined['n_nodes']} nodes, {combined['edge_index'].shape[1]} edges")
 
+    # Real, separate storage per temporal_type -- "gru" (the original,
+    # default architecture, used in every real run this project has
+    # produced so far) keeps the existing, unchanged path for backward
+    # compatibility; "lstm"/"cnn" get their own real subfolder, so a
+    # later comparison run never silently overwrites a different
+    # architecture's checkpoint, and each type's own archive/ history
+    # (see the archiving logic below) stays genuinely separate too.
     output_dir = args.data_root / "spatiotemporal_gnn"
+    if args.temporal_type != "gru":
+        output_dir = output_dir / args.temporal_type
     output_dir.mkdir(parents=True, exist_ok=True)
     try:
         plot_selected_subgraph(basin_data, combined, output_dir / "selected_subgraph.png")
@@ -1481,12 +1897,95 @@ def main() -> None:
     }
     print(f"Real days-since-last-observation channel added for: {list(dt_tensors.keys())}")
 
+    # Real, per-node fill values -- CRITICAL, CONFIRMED FIX (see
+    # prepare_graph_training_windows' own docstring): plain 0.0 (the
+    # global-standardized "neutral" fill) unstandardizes to the GLOBAL
+    # mean, which is wildly non-neutral for any real, small station far
+    # below that global average -- confirmed directly, traced via a
+    # real false-spike investigation, to a real ~60x input distortion
+    # at one real station during a real data gap. Computed from the
+    # real TRAINING period only (same leakage discipline as
+    # discharge_mean/std themselves), on the ALREADY-standardized
+    # dynamic_tensors, so each value is a per-node offset in the same
+    # global-standardized units the rest of the tensor already uses.
+    #
+    # MEDIAN, not mean, per a real, confirmed finding: the per-node
+    # MEAN fill value for one real station (H605641401) was itself
+    # still ~30x that station's real low-flow values, because real
+    # discharge is right-skewed and rare, real flood days pull the
+    # mean well above a real "typical" day. See
+    # compute_per_node_historical_median's own docstring.
+    train_end_ts_for_fill = pd.Timestamp(args.train_end)
+    train_date_mask_for_fill = np.array([d <= train_end_ts_for_fill for d in dates])
+    per_node_fill_values = {
+        var: compute_per_node_historical_median(tensor[:, train_date_mask_for_fill])
+        for var, tensor in dynamic_tensors.items()
+    }
+
+    # CONFIRMED FIX #2 -- forward-fill, preferred over the per-node
+    # median above whenever a real prior observation exists to carry
+    # forward. A station-wide median is only neutral for a node's
+    # TYPICAL day; it's the wrong reference for a gap that falls
+    # inside a real, ongoing low- or high-flow episode (confirmed
+    # directly: H605641401's real per-node MEDIAN fill, 1142.0, was
+    # still ~17x its real low-flow values, 57-70 L/s, during a real
+    # gap inside a real low-flow episode -- and a direct causal
+    # ablation confirmed the missingness-flag mechanism alone cannot
+    # compensate for a fill value that's simply the wrong magnitude).
+    # Computed on the SAME already-standardized dynamic_tensors as
+    # per_node_fill_values above, over the FULL real date range (not
+    # just the training period) -- unlike per_node_fill_values,
+    # forward-fill needs no train-only-fit discipline at all, since it
+    # only ever looks backward from any given date (see
+    # compute_forward_filled_tensor's own docstring).
+    forward_fill_tensors = {
+        var: compute_forward_filled_tensor(tensor)
+        for var, tensor in dynamic_tensors.items()
+    }
+
     X, Y, Y_mask, anchor_dates = prepare_graph_training_windows(
         combined["x_shared_static"], dynamic_tensors, target_tensor, dates, args.lookback_days, HORIZONS,
-        dt_tensors=dt_tensors,
+        dt_tensors=dt_tensors, per_node_fill_values=per_node_fill_values,
+        forward_fill_tensors=forward_fill_tensors,
     )
     print(f"Training windows: X={X.shape}, Y={Y.shape}")
 
+    # --event-reweight -- ALWAYS build an event_weight array, same shape
+    # as Y's (example, node, horizon) axes, even when the flag is off:
+    # an all-1.0 array makes the weighted loss functions below EXACTLY
+    # reproduce the original unweighted ones (confirmed directly --
+    # weighted_multi_quantile_loss's single grand-weighted-mean equals
+    # multi_quantile_loss's mean-of-per-quantile-means whenever every
+    # quantile has the same real finite-element count, which is always
+    # true here since finiteness is governed by Y's own NaN pattern, not
+    # by which quantile is being scored). This keeps the training loop
+    # below simple -- always call the weighted form, no separate
+    # on/off branch needed there.
+    event_weight = np.ones((len(anchor_dates), combined["n_nodes"], len(HORIZONS)), dtype=np.float32)
+    if args.event_reweight:
+        events_path_train = args.events_path or (args.data_root / "flood_events_discharge.csv")
+        if events_path_train.exists():
+            events_df_train = pd.read_csv(events_path_train)
+            if events_df_train.empty:
+                print(f"--event-reweight requested, but {events_path_train} has no real events -- "
+                      f"run identify_flood_events.py first. Proceeding with neutral weight=1.0 "
+                      f"everywhere (a no-op).")
+            else:
+                event_lookup_train = build_event_lookup(events_df_train)
+                event_weight = build_event_weight_tensor(
+                    combined["station_codes"], anchor_dates, HORIZONS, event_lookup_train,
+                    window_days=args.event_reweight_window_days, boost=args.event_reweight_boost,
+                )
+                n_boosted = int((event_weight > 1.0).sum())
+                print(f"--event-reweight: loaded {len(events_df_train)} real event(s) across "
+                      f"{events_df_train['station_code'].nunique()} station(s); boosted "
+                      f"{n_boosted} (example, node, horizon) value(s) to weight={args.event_reweight_boost} "
+                      f"(within {args.event_reweight_window_days} real day(s) of a real labeled event)")
+        else:
+            print(f"--event-reweight requested, but {events_path_train} doesn't exist -- run "
+                  f"identify_flood_events.py first. Proceeding with neutral weight=1.0 everywhere "
+                  f"(a no-op).")
+
     Y_H = extract_targets_at_anchor_dates(waterlevel_target_tensor, dates, anchor_dates, HORIZONS)
     Y_H_mask = ~np.isnan(Y_H)
     n_real_wl = int(Y_H_mask.sum())
@@ -1553,9 +2052,9 @@ def main() -> None:
         return
 
     train_ds = GraphWindowDataset(X[train_mask], Y[train_mask], Y_mask[train_mask], Y_H[train_mask], Y_H_mask[train_mask],
-                                   forecast_precip[train_mask], forecast_missing[train_mask])
+                                   forecast_precip[train_mask], forecast_missing[train_mask], event_weight[train_mask])
     val_ds = GraphWindowDataset(X[val_mask], Y[val_mask], Y_mask[val_mask], Y_H[val_mask], Y_H_mask[val_mask],
-                                 forecast_precip[val_mask], forecast_missing[val_mask])
+                                 forecast_precip[val_mask], forecast_missing[val_mask], event_weight[val_mask])
     train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True)
     val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False)
 
@@ -1584,17 +2083,25 @@ def main() -> None:
         shared_static_dim=x_shared_static_t.shape[1], gauge_static_dim=x_gauge_static_t.shape[1],
         gauge_dynamic_dim=n_gauge_dynamic_channels, edge_dim=edge_attr_t.shape[1],
         n_basins=len(BASIN_FILE_NAMES), n_targets=2,  # [discharge, water_level]
-        quantiles=QUANTILES,
+        quantiles=QUANTILES, temporal_type=args.temporal_type,
     )
 
     physics_weights = {"confluence": args.confluence_weight, "split_rejoin": args.split_rejoin_weight,
                         "routing": args.routing_weight, "manning": args.manning_weight,
                         "water_balance": args.water_balance_weight,
                         "routing_soft_dtw": args.routing_soft_dtw_weight,
-                        "nse": args.nse_weight}
+                        "nse": args.nse_weight, "kge": args.kge_weight}
     print(f"Physics loss weights: {physics_weights}")
     print()
 
+    # Same real per-temporal_type default location as model.pt itself
+    # -- see --checkpoint-path's own help text for why this is a
+    # SEPARATE real file from model.pt (resumable: carries optimizer
+    # state/epoch/history too, updated every improving epoch, not just
+    # once at the very end of training).
+    checkpoint_path = args.checkpoint_path or (output_dir / "checkpoint.pt")
+    resume_from = checkpoint_path if args.resume else None
+
     # Real precipitation channel index -- None if the real precipitation
     # CSV wasn't fetched for this run, in which case water balance is
     # skipped entirely inside compute_physics_loss, not silently faked.
@@ -1614,6 +2121,10 @@ def main() -> None:
 
     discharge_mean, discharge_std = standardization_stats["discharge"]
     precip_mean, precip_std = standardization_stats.get("precipitation", (0.0, 1.0))
+    # Needed for --use-kge-loss: real KGE requires real, un-standardized
+    # units, confirmed as a real, structural requirement, not a nicety
+    # (see kge_loss_from_pooled_pairs' own docstring).
+    waterlevel_mean, waterlevel_std = standardization_stats.get("waterlevel", (0.0, 1.0))
 
     # Real per-node historical mean, for NSE -- fit on the training
     # period only (same discipline as discharge_mean/std above), from
@@ -1628,6 +2139,19 @@ def main() -> None:
     waterlevel_per_node_mean_t = torch.tensor(
         compute_per_node_historical_mean(waterlevel_target_tensor[:, train_date_mask]), dtype=torch.float32,
     )
+    # For --per-station-relative-loss: real, per-node std, in REAL,
+    # physical L/s units (converted from the standardized-space std by
+    # multiplying by the real, global discharge_std -- std(a*X) =
+    # |a|*std(X), confirmed directly before use, not assumed). Built
+    # after real, direct evidence (check_discharge_quantile_
+    # distribution.py) showed this project's 6 real gauges have wildly
+    # different real scale (Q50 ranging from real ~868 L/s to ~9323
+    # L/s) -- a single, global discharge_std treats a real,
+    # proportionally large error at a small station as much smaller
+    # than the identical real proportional error at a large station.
+    discharge_per_node_std_t = torch.tensor(
+        compute_per_node_historical_std(target_tensor[:, train_date_mask]) * discharge_std, dtype=torch.float32,
+    )
 
     result = train_spatiotemporal_model(
         model, train_loader, val_loader, x_shared_static_t, x_gauge_static_t, is_gauged_t,
@@ -1640,6 +2164,16 @@ def main() -> None:
         soft_dtw_gamma=args.routing_soft_dtw_gamma,
         discharge_per_node_mean=discharge_per_node_mean_t,
         waterlevel_per_node_mean=waterlevel_per_node_mean_t,
+        use_log_space_discharge_loss=args.log_space_discharge_loss,
+        log_discharge_offset=args.log_discharge_offset,
+        use_per_station_relative_loss=args.per_station_relative_loss,
+        discharge_per_node_std=discharge_per_node_std_t,
+        use_kge_loss=args.use_kge_loss,
+        kge_min_points=args.kge_min_points,
+        waterlevel_mean=waterlevel_mean,
+        waterlevel_std=waterlevel_std,
+        checkpoint_path=checkpoint_path,
+        resume_from=resume_from,
     )
 
     model_path = output_dir / "model.pt"
@@ -1658,6 +2192,10 @@ def main() -> None:
         shutil.move(str(model_path), str(archived_path))
         print(f"Archived previous checkpoint to {archived_path}")
     torch.save(result["model_state"], model_path)
+    print(f"Saved final real model (bare state_dict, for evaluate_flood_detection.py etc.) to {model_path}")
+    print(f"(The real resumable checkpoint at {checkpoint_path} -- model+optimizer+epoch+history -- "
+          f"already held these exact same best real weights since epoch {result.get('best_epoch', '?')}; "
+          f"this final save is a convenience copy in the plain format the evaluation scripts expect.)")
     print(f"\nBest validation loss: {result['best_val_loss']:.4f}")
     learned_n = 0.01 + 0.29 * torch.sigmoid(model.manning_n_logit).item()
     print(f"Learned Manning's roughness n = {learned_n:.4f} "

src/graph/dynamic_features.py

@@ -334,6 +334,57 @@ def compress_days_since_last_real(dt):
     return np.log1p(dt).astype(np.float32)
 
 
+def compute_forward_filled_tensor(tensor):
+    """
+    Real forward-filled value per [node, date] position: the most
+    recent real (non-NaN) observation at or before that date, NaN if
+    no real observation has occurred yet anywhere before it. Causally
+    safe by construction -- forward-fill only ever looks backward in
+    time, so unlike a fitted statistic (a per-node mean/median, which
+    needs a "fit on train only" discipline to avoid leaking future
+    information into the fill), this needs no train/val/test split
+    guard at all: a forward-filled value at any date, in any split,
+    only ever depends on dates strictly before it.
+
+    Built specifically as a fill-value SOURCE for real gaps during a
+    real low- (or high-) flow episode, where a station-wide central-
+    tendency fill (compute_per_node_historical_median, in
+    physics_losses.py) is itself the wrong reference point --
+    confirmed directly via a real false-spike investigation: even a
+    real per-node MEDIAN fill (1142.0, for station H605641401) was
+    still ~17x that station's real low-flow values (57-70 L/s) during
+    a real gap that fell IN THE MIDDLE of a real low-flow episode,
+    because a median describes a station's typical day, not
+    necessarily the day immediately preceding a specific gap. A real,
+    direct ablation (zeroing the missingness flag and days-since
+    channel for this exact example) confirmed the flag mechanism alone
+    cannot fix this: even with the model given no signal at all that
+    the value was filled, the median-based fill still produced a
+    predicted 0.95 quantile ~28x the real observed value, because the
+    filled VALUE itself (1142.0) was implausible for the real
+    conditions at that moment, not because the model failed to
+    discount a flagged fill. Forward-fill instead carries the real,
+    most-recently-known value forward, directly informative about
+    real LOCAL, CURRENT conditions in a way a station-wide statistic
+    cannot be.
+
+    Same accumulate-forward pattern already validated in
+    compute_days_since_last_real -- computed over the FULL real date
+    range, not per-window, for the same reason: correctly looks
+    backward past wherever a later lookback window happens to start.
+    """
+    n_nodes, n_dates = tensor.shape
+    is_real = ~np.isnan(tensor)
+    date_indices = np.arange(n_dates)
+
+    idx_where_real = np.where(is_real, date_indices[None, :], -1)
+    last_real_idx = np.maximum.accumulate(idx_where_real, axis=1)
+
+    filled = np.take_along_axis(tensor, np.clip(last_real_idx, 0, None), axis=1)
+    filled = np.where(last_real_idx < 0, np.nan, filled)
+    return filled.astype(np.float32)
+
+
 def build_precipitation_timeseries(
     nodes_df: pd.DataFrame, precip_csv_path: Path, date_range: Tuple[str, str], freq: str = "D",
 ) -> pd.DataFrame:
@@ -357,7 +408,40 @@ def build_precipitation_timeseries(
     precip_df = pd.read_csv(precip_csv_path)
     precip_df["date"] = pd.to_datetime(precip_df["date"])
 
-    assignments = map_stations_to_precipitation_gauges(nodes_df, precip_df[["NUM_POSTE", "LAT", "LON"]])
+    # CRITICAL, CONFIRMED FIX -- exclude real "ghost" stations (present
+    # in the real Météo-France station list, with real LAT/LON, but
+    # ZERO real precip_mm_daily readings across every real row they
+    # have) from the candidate pool BEFORE nearest-neighbor matching.
+    # Confirmed directly against a real daily_precipitation.csv: 56 of
+    # 110 real candidate stations (51%) have precip_mm_daily entirely
+    # NaN -- a real fetch/ingestion gap in fetch_meteofrance_
+    # precipitation.py, not a downstream bug -- yet
+    # map_stations_to_precipitation_gauges has no liveness check, only
+    # a distance check, so any real graph node whose real geographically
+    # -nearest station happened to be one of these 56 real ghosts got
+    # permanently, silently matched to a station that can never
+    # contribute a single real reading. Confirmed as the real, direct
+    # cause of 22 of 27 real gauge stations in one real evaluation run
+    # having ZERO real precipitation anywhere in the full 2013-2026
+    # record -- every one of forward-fill, the per-node median fallback,
+    # AND compress_days_since_last_real's real 365-day sentinel were all
+    # working exactly as designed; there was simply never a real
+    # observation for any of them to find. Filtering here, once, before
+    # the nearest-neighbor search, routes every real node to the
+    # nearest real LIVE station instead -- a real, if sometimes farther,
+    # station that can actually contribute real data, which is strictly
+    # better than a real ghost at zero distance.
+    stations_with_real_data = precip_df.loc[precip_df["precip_mm_daily"].notna(), "NUM_POSTE"].unique()
+    n_candidates_before = precip_df["NUM_POSTE"].nunique()
+    precip_stations_live = precip_df[precip_df["NUM_POSTE"].isin(stations_with_real_data)]
+    n_candidates_after = precip_stations_live["NUM_POSTE"].nunique()
+    if n_candidates_after < n_candidates_before:
+        print(f"build_precipitation_timeseries: excluding {n_candidates_before - n_candidates_after} of "
+              f"{n_candidates_before} real candidate precipitation station(s) with ZERO real "
+              f"precip_mm_daily readings from nearest-neighbor matching, leaving "
+              f"{n_candidates_after} real live station(s).")
+
+    assignments = map_stations_to_precipitation_gauges(nodes_df, precip_stations_live[["NUM_POSTE", "LAT", "LON"]])
 
     start, end = date_range
     full_index = pd.date_range(start, end, freq=freq)

src/graph/flood_events.py

@@ -0,0 +1,170 @@
+"""
+Real labeled flood-event bookkeeping -- shared between
+evaluate_flood_detection.py (which USES these events to flag real
+false/true positives after training) and train_spatiotemporal_gnn.py
+(which, as of the event-reweighted loss added here, USES the same real
+events to upweight training examples near them). Moved out of
+evaluate_flood_detection.py into its own module specifically so
+train_spatiotemporal_gnn.py can import it too, without creating a
+circular import -- evaluate_flood_detection.py already imports FROM
+scripts.training.train_spatiotemporal_gnn (shared data-pipeline functions), so
+the reverse import would have failed; this module has no dependency on
+either script, so both can import it safely.
+
+events_df is identify_flood_events.py's own output (a CSV with columns
+station_code, start_date, end_date, threshold) -- both scripts treat
+this the same way: real, labeled, human-verifiable flood windows, not
+a heuristic re-derived at import time.
+"""
+from typing import Dict, List
+
+import numpy as np
+import pandas as pd
+
+
+def build_event_lookup(events_df: pd.DataFrame) -> dict:
+    """
+    Real event date ranges, indexed by station, for fast membership
+    checks -- {station_code: [(start_date, end_date, threshold), ...]}.
+    A single station can have several real events across the record, so
+    this stays a list per station rather than assuming at most one.
+    """
+    lookup = {}
+    for _, row in events_df.iterrows():
+        lookup.setdefault(row["station_code"], []).append(
+            (pd.Timestamp(row["start_date"]), pd.Timestamp(row["end_date"]), row["threshold"])
+        )
+    return lookup
+
+
+def is_real_event_day(station_code: str, date: pd.Timestamp, event_lookup: dict):
+    """
+    Returns the real threshold if `date` falls within a real event for
+    this station, else None -- distinguishing "not an event" from
+    "no threshold defined" matters for how the caller should treat it,
+    so this returns the threshold itself rather than a bare boolean.
+    """
+    for start, end, threshold in event_lookup.get(station_code, []):
+        if start <= date <= end:
+            return threshold
+    return None
+
+
+def days_to_nearest_real_event(station_code: str, date: pd.Timestamp, event_lookup: dict) -> "float":
+    """
+    Real, signed distance in days from `date` to the nearest real event
+    window for this station -- 0 if `date` falls inside one. Built to
+    distinguish a real false positive that's a near-miss (a day or two
+    from an actual real event) from one with no real event anywhere
+    nearby, which are genuinely different situations worth telling
+    apart when inspecting false positives directly.
+
+    Returns float("inf") if this station has no real events at all.
+    """
+    best = float("inf")
+    for start, end, _ in event_lookup.get(station_code, []):
+        if start <= date <= end:
+            return 0.0
+        gap = min(abs((date - start).days), abs((date - end).days))
+        best = min(best, gap)
+    return best
+
+
+def build_station_thresholds(events_df: pd.DataFrame) -> dict:
+    """
+    One real threshold per station -- identify_flood_events.py computes
+    this once per station (that station's own historical percentile),
+    so every real event row for a given station carries the same value;
+    this collapses that back down to {station_code: threshold}.
+
+    This threshold is what gets compared against EVERY day's
+    prediction, event or not -- comparing only on real event days (and
+    treating every other day as an automatic non-flag) would make false
+    positives structurally impossible and precision meaningless by
+    construction, not just optimistic.
+    """
+    return events_df.groupby("station_code")["threshold"].first().to_dict()
+
+
+def build_event_weight_tensor(
+    station_codes: List[str], anchor_dates: List[pd.Timestamp], horizons: List[int],
+    event_lookup: dict, window_days: int = 5, boost: float = 4.0,
+) -> np.ndarray:
+    """
+    Real, per-(example, node, horizon) sample weight for the SUPERVISED
+    discharge quantile loss -- boost where the real TARGET date (anchor
+    + horizon - 1 days, not the anchor date itself) falls within
+    `window_days` of a real labeled flood event at that station, 1.0
+    (neutral) everywhere else.
+
+    Built as the direct, concrete response to the real, measured
+    problem this project's own diagnostics confirmed: real flood
+    events are rare (125 across the whole 2013-2025 record), so under
+    plain (or log-space) pinball loss, the sheer VOLUME of quiet-day
+    gradient updates dilutes the few real flood-day updates, even
+    though pinball loss's own per-sample asymmetry already weights a
+    flood-day underprediction more heavily than a quiet-day
+    overprediction of the same size -- that per-sample weighting
+    doesn't fix a per-EXAMPLE-COUNT imbalance. Direct, measured
+    evidence this project already has: --check-quantile-calibration
+    showed q0.95 coverage at 0.990 on low-flow days (should be ~0.95)
+    and 0.698 on high-flow days (should also be ~0.95) on the SAME
+    checkpoint -- opposite-direction miscalibration consistent with a
+    model whose learned quantile function is dominated by the
+    overwhelmingly more common quiet-day examples.
+
+    Deliberately restricted to stations that appear in event_lookup at
+    all (i.e. have at least one real labeled event) -- every other node
+    (virtual/confluence nodes, or real gauges with zero labeled events)
+    gets the neutral weight of 1.0 everywhere, without needing any real
+    per-example work, since days_to_nearest_real_event would return
+    float("inf") for them regardless. This keeps the real cost of
+    building this tensor proportional to (stations with events) x
+    (real examples) x (real horizons), not the full node count -- with
+    ~4,500 nodes but only a handful of real gauges ever having labeled
+    events, this is a real, large, deliberate savings, not a premature
+    optimization.
+
+    Args:
+        station_codes: this run's real, ordered node->station-code list
+            (combined["station_codes"]), same ordering Y's node axis
+            uses.
+        anchor_dates: this run's real anchor dates, same ordering X/Y's
+            example axis uses (from prepare_graph_training_windows).
+        horizons: this run's real HORIZONS list, same ordering Y's
+            horizon axis uses.
+        event_lookup: from build_event_lookup(events_df).
+        window_days: how many real days from a labeled event's start/
+            end the boost still applies -- 5 chosen as a real, moderate
+            window (identify_flood_events.py's own docstring notes real
+            event durations already run 3-10 days, so this catches the
+            real rising/falling limb immediately around a labeled
+            event, not just the event's own days).
+        boost: the real weight multiplier applied to boosted examples
+            (an untouched example keeps weight 1.0). 4.0 chosen as a
+            real, moderate value in the 3-5x range this project's own
+            discussion landed on -- large enough to meaningfully shift
+            gradient share toward flood-adjacent examples, not so large
+            it risks destabilizing training the way an aggressive
+            reweighting scheme could. Treat as a real, tunable
+            hyperparameter, not a fixed constant -- compare against
+            boost=1.0 (a no-op, exactly reproducing the unweighted
+            loss) as the baseline for any real run using this.
+    """
+    n_examples = len(anchor_dates)
+    n_nodes = len(station_codes)
+    n_horizons = len(horizons)
+    weight = np.ones((n_examples, n_nodes, n_horizons), dtype=np.float32)
+
+    stations_with_events = [code for code in station_codes if event_lookup.get(code)]
+    if not stations_with_events:
+        return weight
+
+    for code in stations_with_events:
+        node_idx = station_codes.index(code)
+        for ex_idx, anchor in enumerate(anchor_dates):
+            for h_idx, h in enumerate(horizons):
+                target_date = anchor + pd.Timedelta(days=h - 1)
+                if days_to_nearest_real_event(code, target_date, event_lookup) <= window_days:
+                    weight[ex_idx, node_idx, h_idx] = boost
+    return weight

src/graph/physics_losses.py

@@ -801,6 +801,76 @@ def compute_per_node_historical_mean(target_tensor: np.ndarray) -> np.ndarray:
     return means.astype(np.float32)
 
 
+def compute_per_node_historical_median(target_tensor: np.ndarray) -> np.ndarray:
+    """
+    Per-node MEDIAN of whatever real (non-NaN) observations exist in
+    target_tensor -- deliberately separate from
+    compute_per_node_historical_mean, not a replacement for it.
+    NSE's formula (1 - sum((obs-pred)^2)/sum((obs-mean)^2)) mathematically
+    requires the real MEAN specifically -- swapping mean for median
+    there would silently stop computing NSE at all. This function
+    exists for a genuinely different real use: a robust, real, per-
+    node FILL VALUE for prepare_graph_training_windows' missing-value
+    handling.
+
+    Real, confirmed reason median, not mean, is the right choice there:
+    a real, direct trace showed the per-node MEAN fill value for one
+    real, low-flow station (H605641401) was itself ~2023 L/s -- still
+    roughly 30x that station's real low-flow values (real ~57-70 L/s)
+    -- because real discharge is right-skewed (rare, real high-flow
+    events pull the real mean well above what a "typical" day actually
+    looks like). The real median is far more robust to that real skew,
+    landing much closer to a station's real, typical day.
+
+    Nodes with zero real observations anywhere get NaN, not a
+    fabricated 0 -- the caller (prepare_graph_training_windows) falls
+    back to the original, global 0.0 fill for such nodes, same as
+    compute_per_node_historical_mean's own NaN nodes already do.
+    """
+    import warnings
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", message="All-NaN slice encountered")
+        medians = np.nanmedian(target_tensor, axis=1)
+    return medians.astype(np.float32)
+
+
+def compute_per_node_historical_std(target_tensor: np.ndarray, epsilon: float = 1e-3) -> np.ndarray:
+    """
+    Per-node std of whatever real (non-NaN) observations exist in
+    target_tensor -- same real, training-period-only discipline as
+    compute_per_node_historical_mean. Built specifically for real,
+    per-station-relative quantile loss normalization: real, direct
+    evidence (check_discharge_quantile_distribution.py) showed this
+    project's 6 real gauges have WILDLY different real discharge
+    scales and skew (Q50/Q01 ranging from 1.4x at one real station to
+    literally infinite -- real zero -- at another), so a single,
+    GLOBAL discharge_std treats a real, proportionally large error at
+    a small station and a real, proportionally tiny error at a large
+    station as if they carried the same real weight in the loss, which
+    they do not.
+
+    A real, small floor (epsilon) prevents division by a near-zero std
+    for a station with real, genuinely low variability (e.g. the two
+    Eure stations found to be nearly flat at low flow) -- without this,
+    such a station's ALREADY small real errors would be divided by an
+    even smaller real denominator, exploding into a disproportionate,
+    destabilizing loss contribution -- the same class of numerical
+    risk KGE's real bias-ratio term already showed us directly.
+
+    Nodes with zero real observations, or only one, get NaN (std is
+    undefined from a single point) -- callers must handle this the
+    same way nse_loss already handles a NaN per_node_mean.
+    """
+    import warnings
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", message="Degrees of freedom <= 0 for slice")
+        stds = np.nanstd(target_tensor, axis=1)
+    stds = np.where(stds < epsilon, epsilon, stds)
+    n_real_per_node = np.sum(~np.isnan(target_tensor), axis=1)
+    stds = np.where(n_real_per_node < 2, np.nan, stds)
+    return stds.astype(np.float32)
+
+
 def nse_loss(Y_true, Y_pred, per_node_mean, epsilon: float = 1e-6):
     """
     1 - NSE, framed as a loss to minimize. A different signal from
@@ -851,6 +921,90 @@ def nse_loss(Y_true, Y_pred, per_node_mean, epsilon: float = 1e-6):
     return float(numerator / denominator)
 
 
+def kge_loss_from_pooled_pairs(obs, pred, epsilon: float = 1e-6, max_loss: float = 20.0):
+    """
+    1 - KGE (Kling-Gupta Efficiency), framed as a loss to minimize --
+    a real, temporally-meaningful replacement for nse_loss, not a
+    cross-station approximation. KGE's correlation term (r) needs
+    multiple real, PAIRED (observed, predicted) points for the SAME
+    node to mean anything -- unlike nse_loss's formula, which only
+    needs a node's own pre-computed historical mean and works fine
+    within a single, sparse per-example training step, a real
+    correlation genuinely requires several real points. This function
+    expects the CALLER to have already gathered those real pairs for
+    one node, pooled across multiple real anchor dates within a batch
+    (see the real pooling logic in train_spatiotemporal_gnn.py's batch
+    loop) -- it does not gather or mask NaN itself, unlike every other
+    loss in this file, because by the time this function is called,
+    only real, already-matched pairs should remain.
+
+    KGE = 1 - sqrt((r-1)^2 + (alpha-1)^2 + (beta-1)^2), where:
+      r     = real Pearson correlation between obs and pred (timing)
+      alpha = std(pred) / std(obs)  (variability ratio)
+      beta  = mean(pred) / mean(obs)  (bias ratio)
+
+    CRITICAL, CONFIRMED REQUIREMENT: obs/pred MUST be in real,
+    UN-STANDARDIZED physical units (real L/s, real cm) here, NOT
+    standardized (z-scored) values -- confirmed directly as a real,
+    structural bug, not a rare edge case: on realistic standardized-
+    space-like pooled samples (mean~0, std~1, exactly what a small,
+    sparse real pool of z-scored values looks like), beta = pred_mean
+    / obs_mean exploded to a worst-case loss of 8,260 across 1000
+    trials, with 4% exceeding loss=10 -- versus a worst case of 4.07,
+    zero trials over 10, on realistic REAL, physical-unit discharge
+    values. Standardization deliberately centers data at mean~0, which
+    is exactly the condition that makes a mean-ratio term numerically
+    unstable; real, physical units don't have that problem, since a
+    real station's real discharge is never centered near zero. The
+    caller (train_spatiotemporal_gnn.py) un-standardizes before pooling
+    for exactly this reason -- do not call this on standardized values.
+
+    Args:
+        obs, pred: 1-D, same length, already real (no NaN), in REAL,
+            UN-STANDARDIZED physical units -- every real (observed,
+            predicted) pair for ONE node, pooled across however many
+            real anchor dates/horizons contributed a real observation
+            within the current batch.
+        epsilon: prevents division by zero when a node's real pooled
+            std or mean is at or near zero.
+        max_loss: a real, measured defensive ceiling, not a guess --
+            even in real, physical units, a small pooled sample
+            combined with a genuinely bad prediction can still produce
+            a real, large loss (measured worst case 28.92 across 3000
+            trials including deliberately bad predictions, with only
+            0.5% exceeding 20) -- this is a second, independent layer
+            of protection on top of using real units, not a substitute
+            for it.
+
+    Returns:
+        Scalar loss, clamped to [0, max_loss]. Callers should require
+        at least ~3-5 real pooled points before calling this (checked
+        by the caller, not here) -- a correlation from 2 points is not
+        real evidence of timing skill, it is guaranteed to be +-1 or
+        undefined.
+    """
+    is_torch = _HAS_TORCH and isinstance(obs, torch.Tensor)
+
+    if is_torch:
+        obs_mean, pred_mean = obs.mean(), pred.mean()
+        obs_std, pred_std = obs.std(unbiased=False), pred.std(unbiased=False)
+        cov = ((obs - obs_mean) * (pred - pred_mean)).mean()
+        r = cov / (obs_std * pred_std + epsilon)
+        alpha = pred_std / (obs_std + epsilon)
+        beta = pred_mean / (obs_mean + epsilon)
+        kge = 1.0 - torch.sqrt((r - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2)
+        return torch.clamp(1.0 - kge, min=0.0, max=max_loss)
+
+    obs_mean, pred_mean = obs.mean(), pred.mean()
+    obs_std, pred_std = obs.std(), pred.std()
+    cov = ((obs - obs_mean) * (pred - pred_mean)).mean()
+    r = cov / (obs_std * pred_std + epsilon)
+    alpha = pred_std / (obs_std + epsilon)
+    beta = pred_mean / (obs_mean + epsilon)
+    kge = 1.0 - np.sqrt((r - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2)
+    return float(np.clip(1.0 - kge, 0.0, max_loss))
+
+
 # ---------------------------------------------------------------------------
 # 7. Gated spatial smoothness (Dirichlet energy / graph Laplacian) --
 #    only where continuity is actually expected

src/graph/pinball_loss.py

@@ -17,7 +17,7 @@ amount, which is exactly the incentive missing from MSE: a model
 trained with this loss is pushed toward actually covering the real
 tail, not toward the safe average.
 """
-from typing import List
+from typing import List, Optional
 
 import numpy as np
 
@@ -88,50 +88,276 @@ def multi_quantile_loss(y_true, y_pred_quantiles, quantiles: List[float]):
     return total / n_real_terms
 
 
-def weighted_multi_quantile_loss(y_true, y_pred_quantiles, quantiles: List[float], recency_weight, alpha: float = 1.0):
+def log_space_multi_quantile_loss(
+    y_true_standardized, y_pred_quantiles_standardized, quantiles: List[float],
+    real_mean: float, real_std: float, log_offset: Optional[float] = None,
+    sample_weight=None,
+):
     """
-    Same real pinball-loss sum as multi_quantile_loss, but each real
-    element is additionally weighted by 1 + alpha*log(1+dt) before
-    averaging -- forcing more real loss-weight onto observations that
-    follow a longer real gap in that node's own history, so the model
-    is pushed to actually close a real drift accumulated over a gap
-    rather than being scored the same regardless of how stale its
-    recent information was.
-
-    A REAL, HONEST OVERLAP WARNING, not a minor footnote: this project
-    already built and wired in a days-since-last-real-observation
-    channel as a direct INPUT feature (see dynamic_features.py's
-    compute_days_since_last_real / compress_days_since_last_real). That
-    gives the model itself the same underlying real information and
-    lets it learn how to use staleness; this function instead FORCES
-    more loss-weight onto post-gap examples regardless of whether the
-    model finds that useful. Two different mechanisms carrying the same
-    real underlying signal, in directions that aren't guaranteed to be
-    aligned -- this is a real, plausible risk of diluting rather than
-    reinforcing the existing signal, not a hypothetical concern. Treat
-    alpha=0 (a no-op, recovering exactly multi_quantile_loss's own
-    behavior) as the real baseline to compare against before trusting
-    any alpha>0 run's results -- an ablation, not an assumption either
-    mechanism helps.
+    Same real pinball-loss mechanism as multi_quantile_loss, but
+    computed in log(real discharge + log_offset) space instead of
+    standardized (z-scored) space -- built specifically to address a
+    real, measured problem: real evaluation showed high-flow PBIAS at
+    25.0% vs. -1.8% at low flow for one real station, confirmed as
+    peak-specific, not a general low-flow issue. In real, linear
+    units, a large absolute error during a real 30,000 L/s peak swamps
+    a proportionally IDENTICAL relative error during a real 500 L/s
+    baseline day, even under pinball loss's linear (not squared)
+    penalty -- log-space compresses that gap so a given real
+    PROPORTIONAL error contributes comparable loss regardless of
+    whether it happens at low or high flow, directly targeting the
+    confirmed peak-specific bias.
+
+    log_offset MATTERS, CONFIRMED DIRECTLY: the original log1p (offset
+    fixed at 1.0) turned out to weight a low-flow error ~28x more than
+    an identical high-flow error (measured on real station-scale
+    values), and a real training run with that offset produced a real,
+    severe regression -- low-flow NSE collapsed to -116.7, PBIAS to
+    -102.7%, at the same real station whose high-flow bias this was
+    built to fix. A larger, real, data-derived offset softens that
+    ratio (measured directly: 0.25*discharge_std -> ~10x, not
+    eliminating the low-flow priority, but no longer as extreme).
+    log_offset defaults to None, which resolves to 0.25*real_std here
+    -- a real, tested starting point, not the original, now-known-
+    risky offset=1.0 default. HONEST CAVEAT: this addresses one real,
+    understood mechanism (log-space's near-zero sensitivity) -- it
+    does NOT fully explain why kge_loss_from_pooled_pairs, a genuinely
+    different mechanism, produced a similar real low-flow regression
+    independently. Treat this as a real, testable improvement, not a
+    confirmed, complete fix, until re-evaluated the same
+    stratified-low/high-flow way the original regression was found.
+
+    SCOPED TO THIS FUNCTION ONLY -- deliberately not a project-wide
+    change. confluence_mass_balance_loss and water_balance_loss need
+    real, ADDITIVE, linear-space discharge (Q_downstream ~=
+    sum(Q_upstream)); log(a)+log(b) != log(a+b), so log-transforming
+    discharge throughout this project would break the physical meaning
+    of every real mass-conservation constraint already built and
+    validated. Everything else keeps operating on real, linear
+    discharge exactly as before; only the supervised term changes.
+
+    Args:
+        y_true_standardized: real target, in the SAME standardized
+            (z-scored) space the model already operates in -- this
+            function does the un-standardization internally, the
+            caller doesn't need to pass real, un-standardized values.
+        y_pred_quantiles_standardized: [..., n_quantiles], same
+            standardized space.
+        quantiles: quantile levels, e.g. [0.5, 0.9, 0.95, 0.99].
+        real_mean, real_std: this run's real discharge_mean/
+            discharge_std (from standardize_dynamic_tensors, fit on
+            the real training period only) -- needed to recover real,
+            physical L/s values before the log transform, since log
+            has no valid meaning applied directly to a z-score.
+        log_offset: the real "c" in log(x + c). None resolves to
+            0.25*real_std. Larger values soften low-flow sensitivity
+            (real, measured trade-off, not a free parameter -- see
+            this docstring's own measured ratios).
+        sample_weight: optional, same meaning as
+            weighted_multi_quantile_loss's own sample_weight -- e.g.
+            train_spatiotemporal_gnn.py's --event-reweight combines
+            with log-space loss by passing the same event-proximity
+            weight tensor through here. None (the default) reproduces
+            the original, unweighted behavior exactly.
+    """
+    is_torch = _HAS_TORCH and isinstance(y_true_standardized, torch.Tensor)
+    if log_offset is None:
+        log_offset = 0.25 * real_std
+
+    y_true_real = y_true_standardized * real_std + real_mean
+    y_pred_real = y_pred_quantiles_standardized * real_std + real_mean
+
+    # Real discharge is never negative; log is undefined at/below
+    # -log_offset. This clamp is NOT the sole force keeping predictions
+    # non-negative the way Manning's n's old hard clamp was (a single,
+    # shared, permanently-reused scalar that could get stuck outside
+    # its range) -- the pinball loss itself, even computed in log-
+    # space, naturally pulls every real prediction toward real, non-
+    # negative observed discharge, and each (node, horizon, quantile)
+    # value here comes from an independent forward pass, so an
+    # occasional real clamp doesn't create the same permanent-zero-
+    # gradient trap.
+    if is_torch:
+        y_true_log = torch.log(torch.clamp(y_true_real, min=0.0) + log_offset)
+        y_pred_log = torch.log(torch.clamp(y_pred_real, min=0.0) + log_offset)
+    else:
+        y_true_log = np.log(np.clip(y_true_real, 0.0, None) + log_offset)
+        y_pred_log = np.log(np.clip(y_pred_real, 0.0, None) + log_offset)
+
+    if sample_weight is None:
+        return multi_quantile_loss(y_true_log, y_pred_log, quantiles)
+    return weighted_multi_quantile_loss(y_true_log, y_pred_log, quantiles, sample_weight)
+
+
+def per_station_relative_multi_quantile_loss(
+    y_true_standardized, y_pred_quantiles_standardized, quantiles: List[float],
+    real_mean: float, real_std: float, per_node_std,
+    sample_weight=None,
+):
+    """
+    Same real pinball-loss mechanism as multi_quantile_loss, but
+    normalizes each real node's real discharge by THAT NODE's OWN real
+    historical std before computing pinball loss, instead of the
+    single, GLOBAL discharge_std every other version of this loss uses.
+
+    Built after real, direct evidence
+    (check_discharge_quantile_distribution.py) showed this project's 6
+    real gauges have WILDLY different real scale and skew -- Q50/Q01
+    ranged from 1.4x at one real station (nearly flat at low flow) to
+    literally infinite (real zero) at another, and Q99/Q50 ranged from
+    4.3x to 12.0x. Under the existing GLOBAL standardization, a real,
+    proportionally large error at a small station (e.g. real Q50~868
+    L/s) contributes a much smaller loss than the identical real
+    PROPORTIONAL error at a large station (e.g. real Q50~9323 L/s),
+    simply because both get divided by the same global discharge_std
+    -- the loss implicitly favors getting the large stations right,
+    not a deliberate, reasoned choice.
+
+    Dividing by each node's own real per_node_std puts every real
+    station's discharge into a comparable, station-relative unit --
+    a real error that's proportionally severe for a given station
+    contributes comparable loss regardless of whether that station
+    is large or small.
+
+    SCOPED TO THIS FUNCTION ONLY -- deliberately not a project-wide
+    change, same discipline as log_space_multi_quantile_loss.
+    confluence_mass_balance_loss and water_balance_loss need real,
+    ADDITIVE, linear-space discharge in the ORIGINAL global scale
+    (Q_downstream ~= sum(Q_upstream)); per-node-relative values aren't
+    additive across nodes the way real physical discharge is, so this
+    transform would break the physical meaning of every real
+    mass-conservation constraint if applied there. Everything else
+    keeps operating on real, linear, globally-standardized discharge
+    exactly as before; only this one supervised term changes.
+
+    Args:
+        y_true_standardized: real target, standardized (z-scored)
+            space, [n_nodes, n_horizons] or similar -- last dim
+            broadcasts against per_node_std via its own leading node
+            dimension.
+        y_pred_quantiles_standardized: [..., n_quantiles], same
+            standardized space, same leading node dimension.
+        quantiles: quantile levels, e.g. [0.5, 0.9, 0.95, 0.99].
+        real_mean, real_std: this run's real, GLOBAL discharge_mean/
+            discharge_std (from standardize_dynamic_tensors) -- needed
+            to recover real, physical L/s values first.
+        per_node_std: [n_nodes] real, per-node historical std (from
+            compute_per_node_historical_std, fit on the training
+            period only) -- real NaN for any node with fewer than 2
+            real training-period observations, which correctly
+            propagates to NaN loss for that node's terms, excluded by
+            multi_quantile_loss's own real NaN-masking, the same way a
+            NaN per_node_mean already gets handled for NSE.
+        sample_weight: optional, same meaning as
+            weighted_multi_quantile_loss's own sample_weight. None (the
+            default) reproduces the original, unweighted behavior
+            exactly.
+    """
+    is_torch = _HAS_TORCH and isinstance(y_true_standardized, torch.Tensor)
+
+    y_true_real = y_true_standardized * real_std + real_mean
+    y_pred_real = y_pred_quantiles_standardized * real_std + real_mean
+
+    # Broadcast per_node_std ([n_nodes]) against y_true_real's
+    # [n_nodes, n_horizons, ...] shape -- reshape to add trailing
+    # singleton dims matching however many extra dims y_true_real has
+    # beyond the leading node dimension.
+    #
+    # CRITICAL, REAL, CONFIRMED FIX: replace NaN entries in per_node_std
+    # (real, ungauged/virtual nodes with no real training-period
+    # discharge at all -- compute_per_node_historical_std correctly
+    # returns NaN for these) with a safe, finite placeholder BEFORE the
+    # division, not after. A real training run confirmed this is NOT
+    # optional: dividing by a real NaN produces a NaN y_pred_relative at
+    # those real node positions, and even though multi_quantile_loss's
+    # own real masking correctly excludes those NaN positions from the
+    # FORWARD loss sum, PyTorch's backward pass still computes
+    # d(NaN)/d(input) for every real parameter that touched that
+    # division -- NaN * 0 is NaN, not 0, in gradient computation, a
+    # real, well-known autograd pitfall. Confirmed directly: a real
+    # training run showed EVERY SINGLE model parameter with a real NaN
+    # gradient from epoch 1, batch 1 onward, tracing back to exactly
+    # this division. The placeholder value itself doesn't matter for
+    # correctness (any finite, positive number works, since the
+    # corresponding real Y_true is ALSO NaN at these same node
+    # positions and gets excluded from the forward sum regardless) --
+    # 1.0 is used here for simplicity and because it can never itself
+    # be zero.
+    if is_torch:
+        safe_per_node_std = torch.where(torch.isnan(per_node_std), torch.ones_like(per_node_std), per_node_std)
+    else:
+        safe_per_node_std = np.where(np.isnan(per_node_std), 1.0, per_node_std)
+
+    if is_torch:
+        true_extra_dims = y_true_real.dim() - 1
+        pred_extra_dims = y_pred_real.dim() - 1
+        node_std_true = safe_per_node_std.reshape(-1, *([1] * true_extra_dims))
+        node_std_pred = safe_per_node_std.reshape(-1, *([1] * pred_extra_dims))
+    else:
+        true_extra_dims = y_true_real.ndim - 1
+        pred_extra_dims = y_pred_real.ndim - 1
+        node_std_true = safe_per_node_std.reshape(-1, *([1] * true_extra_dims))
+        node_std_pred = safe_per_node_std.reshape(-1, *([1] * pred_extra_dims))
+
+    y_true_relative = y_true_real / node_std_true
+    y_pred_relative = y_pred_real / node_std_pred
+
+    if sample_weight is None:
+        return multi_quantile_loss(y_true_relative, y_pred_relative, quantiles)
+    return weighted_multi_quantile_loss(y_true_relative, y_pred_relative, quantiles, sample_weight)
+
+
+def weighted_multi_quantile_loss(y_true, y_pred_quantiles, quantiles: List[float], sample_weight, alpha: float = 1.0):
+    """
+    Same pinball-loss sum as multi_quantile_loss, but each element is
+    additionally weighted before averaging -- a generic weighted
+    pinball average, not tied to any one specific weighting scheme.
+    Two real, distinct uses in this project so far:
+
+    (1) Recency weighting (the original use this was built for): sample_weight
+        = 1 + alpha*log(1+dt) per node, forcing more loss-weight onto
+        observations following a longer gap in that node's own
+        history. Overlap warning: a days-since-last-real-observation
+        channel is already wired in as a direct input feature
+        (dynamic_features.py's compute_days_since_last_real /
+        compress_days_since_last_real), which lets the model learn how
+        to use staleness itself. This function instead forces more
+        loss-weight onto post-gap examples regardless of whether the
+        model finds that useful -- two different mechanisms carrying
+        the same underlying signal, not guaranteed to be aligned.
+
+    (2) Event-proximity weighting (train_spatiotemporal_gnn.py's
+        --event-reweight): sample_weight = build_event_weight_tensor's
+        output, boosting examples near a real labeled flood event.
+        Built to directly counteract real, measured sample-count
+        dilution -- real flood events are rare (125 across the whole
+        2013-2025 record), so the sheer VOLUME of quiet-day gradient
+        updates dominates training even though pinball loss's own
+        per-sample asymmetry already weights a flood-day
+        underprediction more heavily than a same-size quiet-day
+        overprediction; that per-sample weighting doesn't fix a
+        per-example-COUNT imbalance the way this does.
+
+    Treat sample_weight of all-1.0 (a no-op, exactly reproducing
+    multi_quantile_loss) as the baseline to compare any real weighting
+    scheme against, rather than assuming either one helps by default.
 
     Args:
         y_true, y_pred_quantiles, quantiles: same as multi_quantile_loss.
-        recency_weight: [n_nodes] (or broadcastable against y_true's
-            leading node dimension) -- real, already-computed
-            1 + alpha*log(1+dt) values per node. Passed in already
-            computed, not recomputed here, so the caller controls
-            exactly which real dt this reflects (e.g. dt as of the
-            anchor date, not some other point).
-        alpha: real scaling on the log(1+dt) term, applied by the
-            CALLER when building recency_weight -- kept as a parameter
-            here too only for direct-call convenience; if
-            recency_weight is passed pre-built, this argument has no
-            further effect and is not used inside this function.
+        sample_weight: either [n_nodes] (broadcast against y_true's
+            leading node dimension) or full-shaped
+            (same shape as y_true, e.g. [n_nodes, n_horizons]) --
+            already-computed per-element weights.
+        alpha: scaling on the log(1+dt) term for the recency-weighting
+            use case, applied by the caller when building that
+            sample_weight -- kept here only for direct-call
+            convenience; has no effect if sample_weight is pre-built
+            (as it always is for event-proximity weighting).
     """
     is_torch = _HAS_TORCH and isinstance(y_true, torch.Tensor)
-    weight_bcast = recency_weight if (hasattr(recency_weight, "ndim") and recency_weight.ndim == y_true.ndim) else \
-        (recency_weight.view(-1, *([1] * (y_true.ndim - 1))) if is_torch else
-         recency_weight.reshape(-1, *([1] * (y_true.ndim - 1))))
+    weight_bcast = sample_weight if (hasattr(sample_weight, "ndim") and sample_weight.ndim == y_true.ndim) else \
+        (sample_weight.view(-1, *([1] * (y_true.ndim - 1))) if is_torch else
+         sample_weight.reshape(-1, *([1] * (y_true.ndim - 1))))
 
     total = torch.zeros((), dtype=y_true.dtype, device=y_true.device) if is_torch \
         else np.zeros((), dtype=y_true.dtype)

src/graph/spatiotemporal_gnn.py

@@ -49,6 +49,8 @@ def prepare_graph_training_windows(
     x_static: np.ndarray, dynamic_tensors: Dict[str, np.ndarray], target_tensor: np.ndarray,
     dates: List[pd.Timestamp], lookback_days: int, horizons: List[int],
     dt_tensors: Optional[Dict[str, np.ndarray]] = None,
+    per_node_fill_values: Optional[Dict[str, np.ndarray]] = None,
+    forward_fill_tensors: Optional[Dict[str, np.ndarray]] = None,
 ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[pd.Timestamp]]:
     """
     Builds graph-wide training windows -- genuinely different from
@@ -70,6 +72,66 @@ def prepare_graph_training_windows(
     to fill the gap physically, rather than the raw input pretending
     a filled zero is a real observation.
 
+    CRITICAL, CONFIRMED FIX -- per_node_fill_values: plain 0.0 is
+    standardized using the GLOBAL mean/std across every real gauge
+    together, so it's only genuinely "neutral" for a node whose own
+    real scale happens to sit near that global average. Confirmed
+    directly as a real, severe bug, not a theoretical concern: for one
+    real, small station (real Q50~983 L/s, far below the real global
+    discharge_mean~4168.71), a real data gap filled the model's input
+    with a value ~60x larger than that station ever actually sees --
+    traced directly via a real false-spike investigation, and this
+    exact real station showed the worst real low-flow PBIAS measured
+    all project (-240.1%). The missingness flag alone evidently wasn't
+    enough for the model to fully discount a fill value that large and
+    misleading. per_node_fill_values, when provided, gives each real
+    node its OWN real historical mean (in the same global-standardized
+    units the rest of the tensor is already in, via
+    compute_per_node_historical_mean applied to the ALREADY-
+    standardized tensor -- fit on the real training period only, same
+    leakage discipline as discharge_mean/std themselves) as its fill
+    value instead of the single, global 0.0 -- genuinely neutral for
+    THAT node's own real typical range, not the whole network's.
+    A real node with no real training-period observations at all
+    (a true virtual/confluence node, or per_node_fill_values not
+    provided) correctly falls back to the original global 0.0 -- there
+    is no real, per-node information to prefer for such a node.
+
+    forward_fill_tensors, when provided, takes priority over
+    per_node_fill_values as the fill source for a missing timestep.
+    CRITICAL, CONFIRMED FIX #2 -- per_node_fill_values (a station-wide
+    mean or median) is only a genuinely neutral fill for a node's
+    TYPICAL day; it is the wrong reference point for a gap that falls
+    in the middle of a real, ongoing low- (or high-) flow episode.
+    Confirmed directly: even a real per-node MEDIAN fill (1142.0, for
+    station H605641401) was ~17x that station's real low-flow values
+    (57-70 L/s) during a real gap inside a real low-flow episode, and
+    a direct causal ablation (zeroing the missingness flag and
+    days-since channel for this exact example) confirmed the model
+    could not fully compensate: predicted Q0.95 was still ~28x the
+    real observed value even with no "this was filled" signal reaching
+    it at all, because the filled VALUE itself was implausible for the
+    real local conditions at that moment -- not a flag-weighting
+    problem. forward_fill_tensors (see dynamic_features.py's
+    compute_forward_filled_tensor) instead carries the real,
+    most-recently-known value at that specific node forward, which is
+    directly informative about real LOCAL, CURRENT conditions in a way
+    a station-wide statistic structurally cannot be. Needs no
+    train-only-fit discipline (unlike per_node_fill_values) since
+    forward-fill only ever looks backward in time -- no leakage is
+    possible by construction.
+
+    Fill priority per missing timestep, per variable:
+      1. forward_fill_tensors[var] at that [node, date] position, if
+         provided AND real (i.e. some real observation has occurred at
+         that node before that date).
+      2. per_node_fill_values[var] for that node, if provided AND real
+         (station-wide fallback when there's no real prior observation
+         yet for forward-fill to carry forward -- e.g. the very start
+         of a node's real record).
+      3. global 0.0 (original fallback, for a node with no real
+         information at all from either source).
+
     dt_tensors, when provided, adds a THIRD channel per variable: real,
     log-compressed days-since-last-real-observation (see
     dynamic_features.py's compute_days_since_last_real). A binary
@@ -130,7 +192,44 @@ def prepare_graph_training_windows(
         for var in var_names:
             raw_window = dynamic_tensors[var][:, t - lookback_days:t]  # [n_nodes, lookback]
             missing = np.isnan(raw_window)
-            filled = np.where(missing, 0.0, raw_window)
+
+            # Fill priority: forward_fill_tensors (real, local,
+            # most-recent value at THIS node) > per_node_fill_values
+            # (station-wide mean/median, only when forward-fill has no
+            # real prior observation yet to carry) > global 0.0 (no
+            # real information at all). See this function's docstring
+            # ("CRITICAL, CONFIRMED FIX #2") for why forward-fill takes
+            # priority over the station-wide statistic.
+            if forward_fill_tensors is not None and var in forward_fill_tensors:
+                ff_window = forward_fill_tensors[var][:, t - lookback_days:t]  # [n_nodes, lookback]
+                if per_node_fill_values is not None and var in per_node_fill_values:
+                    node_fill = per_node_fill_values[var]
+                    node_fill_broadcast = np.where(
+                        np.isnan(node_fill), 0.0, node_fill,
+                    ).reshape(-1, 1)
+                    stat_fallback = np.broadcast_to(node_fill_broadcast, ff_window.shape)
+                else:
+                    stat_fallback = np.zeros_like(ff_window)
+                # ff_window is NaN only where this node has no real
+                # observation yet anywhere before that date -- fall
+                # back to the station-wide stat (or 0.0) only there.
+                fill_source = np.where(np.isnan(ff_window), stat_fallback, ff_window)
+                filled = np.where(missing, fill_source, raw_window)
+            elif per_node_fill_values is not None and var in per_node_fill_values:
+                # Real, per-node fill values -- [n_nodes], broadcast
+                # across the lookback dimension. A node whose own real
+                # fill value is itself NaN (no real training-period
+                # observations at all for this variable at this node)
+                # correctly falls back to the original, global 0.0 --
+                # np.where below prefers the per-node value only where
+                # it's real.
+                node_fill = per_node_fill_values[var]
+                node_fill_broadcast = np.where(
+                    np.isnan(node_fill), 0.0, node_fill,
+                ).reshape(-1, 1)
+                filled = np.where(missing, node_fill_broadcast, raw_window)
+            else:
+                filled = np.where(missing, 0.0, raw_window)
             window_channels.append(filled)
             window_channels.append(missing.astype(np.float32))
             if dt_tensors is not None:
@@ -275,14 +374,70 @@ if _HAS_TORCH:
             return shared_embed + gauge_embed
 
 
+    class _CausalTemporalCNN(nn.Module):
+        """
+        Real, causal, dilated 1D convolution stack over the lookback
+        window -- an alternative temporal encoder for the real, direct
+        GRU-vs-LSTM-vs-CNN ablation this project is building (same
+        graph layers, same physics losses, same everything else;
+        temporal_type is the only thing that changes). A genuinely
+        different mechanism from a GRU/LSTM (a fixed, dilated
+        receptive field vs. recurrent memory), not a near-duplicate.
+
+        CAUSALITY IS THE REAL, LOAD-BEARING PROPERTY HERE: a GRU/LSTM's
+        forward-only recurrence guarantees a given timestep's hidden
+        state never depends on future inputs. A naive Conv1d with
+        symmetric padding does NOT have that property -- it would let
+        future timesteps leak into the "final" summary this class
+        produces, which is exactly the kind of real look-ahead bug this
+        project has been careful to verify against everywhere else.
+        Fixed here via explicit LEFT-only (causal) padding: pad by
+        (kernel_size-1)*dilation on the left, none on the right, so
+        the output at position t only ever convolves over real inputs
+        at positions <= t. Directly, numerically verified (not just
+        reasoned about) in the same session this was built.
+        """
+        def __init__(self, in_dim: int, out_dim: int, kernel_size: int = 3,
+                     dilations: Tuple[int, ...] = (1, 2, 4, 8)):
+            super().__init__()
+            self.kernel_size = kernel_size
+            self.dilations = list(dilations)
+            self.pads = [(kernel_size - 1) * d for d in self.dilations]
+            convs = []
+            channels = in_dim
+            for dilation in self.dilations:
+                convs.append(nn.Conv1d(channels, out_dim, kernel_size, dilation=dilation))
+                channels = out_dim
+            self.convs = nn.ModuleList(convs)
+
+        def forward(self, seq):
+            # seq: [n_nodes, lookback, in_dim] -> Conv1d wants
+            # [n_nodes, in_dim, lookback] (channels-first).
+            x = seq.transpose(1, 2)
+            for conv, pad in zip(self.convs, self.pads):
+                # Explicit LEFT-only pad -- this is the real causality
+                # mechanism, not Conv1d's own `padding=` argument (which
+                # pads both sides equally and would leak future steps).
+                x = nn.functional.pad(x, (pad, 0))
+                x = torch.relu(conv(x))
+            # [n_nodes, out_dim, lookback] -- the LAST real timestep is
+            # this stack's causal summary of the whole real sequence up
+            # to now, matching a GRU/LSTM's own final-hidden-state
+            # semantics, for a fair, comparable ablation.
+            return x[:, :, -1]
+
+
     class SpatiotemporalGNN(nn.Module):
         """
         Full model: type-aware input encoding (see TypeAwareInputEncoder
         above), two RiverMessagePassing layers per timestep (two hops --
         lets a node see two edges' worth of upstream influence per step,
-        e.g. a confluence's own upstream confluence), a per-node GRU
-        across the lookback window, and a horizon+basin-conditioned
-        decoder with shared weights across both basins.
+        e.g. a confluence's own upstream confluence), a per-node temporal
+        encoder across the lookback window (temporal_type: "gru"
+        (default), "lstm", or "cnn" -- a real, direct ablation of just
+        this one component, everything else identical), and a
+        horizon+basin-conditioned decoder with shared weights across
+        both basins.
         """
 
         def __init__(
@@ -290,6 +445,7 @@ if _HAS_TORCH:
             edge_dim: int = 5, embed_dim: int = 32, hidden_dim: int = 64, gru_hidden: int = 64,
             n_basins: int = 2, basin_embed_dim: int = 8, horizon_embed_dim: int = 8, n_targets: int = 2,
             quantiles: Optional[List[float]] = None, forecast_embed_dim: int = 8,
+            temporal_type: str = "gru",
         ):
             super().__init__()
             self.quantiles = quantiles or [0.5, 0.9, 0.95, 0.99]
@@ -299,7 +455,23 @@ if _HAS_TORCH:
             )
             self.spatial1 = RiverMessagePassing(embed_dim, hidden_dim, edge_dim)
             self.spatial2 = RiverMessagePassing(hidden_dim, hidden_dim, edge_dim)
-            self.temporal = nn.GRU(hidden_dim, gru_hidden, batch_first=True)
+            # temporal_type is a real, direct ablation switch -- "gru"
+            # (default, unchanged behavior), "lstm", or "cnn"
+            # (_CausalTemporalCNN, defined above). Everything else in
+            # this model (graph layers, decoder, physics losses used
+            # during training) stays identical regardless of this
+            # choice, so a real comparison across the three isolates
+            # just the temporal-encoding mechanism, not a mix of
+            # changes.
+            self.temporal_type = temporal_type
+            if temporal_type == "gru":
+                self.temporal = nn.GRU(hidden_dim, gru_hidden, batch_first=True)
+            elif temporal_type == "lstm":
+                self.temporal = nn.LSTM(hidden_dim, gru_hidden, batch_first=True)
+            elif temporal_type == "cnn":
+                self.temporal = _CausalTemporalCNN(hidden_dim, gru_hidden)
+            else:
+                raise ValueError(f"temporal_type must be 'gru', 'lstm', or 'cnn', got {temporal_type!r}")
             self.basin_embed = nn.Embedding(n_basins, basin_embed_dim)
             self.horizon_embed = nn.Linear(1, horizon_embed_dim)
             # Real forecasted precipitation, per node per horizon -- NOT
@@ -399,8 +571,21 @@ if _HAS_TORCH:
                 embeddings.append(h)
             seq = torch.stack(embeddings, dim=1)  # [n_nodes, lookback, hidden_dim]
 
-            _, h_n = self.temporal(seq)
-            node_hidden = h_n.squeeze(0)  # [n_nodes, gru_hidden]
+            # Real per-temporal_type handling -- GRU returns a single
+            # final hidden state, LSTM returns a (h_n, c_n) tuple (real
+            # cell state alongside hidden state, an LSTM-specific
+            # difference from GRU), CNN (_CausalTemporalCNN) already
+            # returns the right [n_nodes, gru_hidden] shape directly, no
+            # further unpacking needed. All three converge to the same
+            # real node_hidden shape the decoder below expects.
+            if self.temporal_type == "gru":
+                _, h_n = self.temporal(seq)
+                node_hidden = h_n.squeeze(0)  # [n_nodes, gru_hidden]
+            elif self.temporal_type == "lstm":
+                _, (h_n, c_n) = self.temporal(seq)
+                node_hidden = h_n.squeeze(0)  # [n_nodes, gru_hidden]
+            else:  # "cnn"
+                node_hidden = self.temporal(seq)  # [n_nodes, gru_hidden]
 
             basin_feat = self.basin_embed(basin_id)  # [n_nodes, basin_embed_dim]