Delete examples

examples/example_pipeline.py

@@ -1,214 +0,0 @@
-"""
-Example pipeline demonstrating Phase 1: Data Fusion and Graph Construction.
-
-This script shows how to use the implemented modules to:
-1. Load hydrometric and SAFRAN data
-2. Clean and validate data
-3. Fuse data sources
-4. Extract basin features
-5. Build physics-informed graph
-6. Prepare tensors for GNN training
-"""
-from pathlib import Path
-from src.config.settings import (
-    WATERSHED_RISLE, WATERSHED_EURE, IDPR_FILE,
-    BASIN_IDS, RANDOM_SEED
-)
-from src.data.loaders.hydrometric import HydrometricLoader
-from src.data.loaders.safran import SAFRANLoader
-from src.data.loaders.shapefile import ShapefileLoader
-from src.data.loaders.idpr import IDPRLoader
-from src.data.extractors.topological import TopologicalExtractor
-from src.data.extractors.karst import KarstExtractor
-from src.fusion.fusion_engine import DataFusionEngine
-from src.graph.builder import HydrologicalGraphBuilder
-from src.preprocessing.tensor_builder import SpatioTemporalTensorBuilder
-from src.utils.helpers import set_seed, get_device, ensure_dir, print_data_summary
-
-
-def main():
-    """Run the complete Phase 1 pipeline."""
-
-    print("="*80)
-    print("PHASE 1: Data Fusion and Graph Construction for Hydrological GNN")
-    print("="*80)
-
-    # Set random seed for reproducibility
-    set_seed(RANDOM_SEED)
-
-    # Get computation device
-    device = get_device(prefer_gpu=True)
-
-    # Create output directories
-    output_dir = ensure_dir(Path("outputs"))
-    data_dir = ensure_dir(output_dir / "processed_data")
-    graph_dir = ensure_dir(output_dir / "graphs")
-    tensor_dir = ensure_dir(output_dir / "tensors")
-
-    # ========================================================================
-    # STEP 1: Load Data
-    # ========================================================================
-    print("\n[STEP 1] Loading data sources...")
-
-    # Note: Update these paths to your actual data locations
-    # This is a template - you'll need to adjust based on your data structure
-
-    # Example: Load hydrometric data
-    # hydro_loader = HydrometricLoader(
-    #     data_path=Path("datasets/hydrometric_data"),
-    #     file_format="csv"
-    # )
-    # hydro_df = hydro_loader.load()
-    # print_data_summary(hydro_df, "Hydrometric Data")
-
-    # Example: Load SAFRAN meteorological data
-    # safran_loader = SAFRANLoader(
-    #     data_path=Path("datasets/safran_data"),
-    #     file_format="csv"
-    # )
-    # safran_df = safran_loader.load()
-    # print_data_summary(safran_df, "SAFRAN Data")
-
-    print("Data loading configured. Update paths to your actual data.")
-
-    # ========================================================================
-    # STEP 2: Extract Basin Features
-    # ========================================================================
-    print("\n[STEP 2] Extracting basin features from shapefiles...")
-
-    # Load watershed shapefiles
-    # risle_loader = ShapefileLoader(WATERSHED_RISLE)
-    # eure_loader = ShapefileLoader(WATERSHED_EURE)
-    # risle_gdf = risle_loader.load()
-    # eure_gdf = eure_loader.load()
-
-    # Extract topological features (if DEM available)
-    # topo_extractor = TopologicalExtractor(dem_path=Path("path/to/dem.tif"))
-    # risle_features = topo_extractor.extract(risle_gdf)
-    # eure_features = topo_extractor.extract(eure_gdf)
-
-    # Extract karst features
-    # idpr_loader = IDPRLoader(IDPR_FILE)
-    # idpr_df = idpr_loader.load()
-    # karst_extractor = KarstExtractor(idpr_data=idpr_df)
-    # karst_features = karst_extractor.extract(risle_gdf)
-
-    print("Basin feature extraction configured.")
-
-    # ========================================================================
-    # STEP 3: Data Fusion
-    # ========================================================================
-    print("\n[STEP 3] Fusing data sources...")
-
-    # Initialize fusion engine
-    fusion_engine = DataFusionEngine(
-        normalize=True,
-        normalization_method="robust",  # Recommended for hydrological data
-        clean_data=True
-    )
-
-    # Fuse datasets
-    # fused_df = fusion_engine.fuse(
-    #     hydrometric_df=hydro_df,
-    #     safran_df=safran_df,
-    #     basin_features=None  # Add extracted basin features here
-    # )
-
-    # Save fused data
-    # fusion_engine.save_fused_data(data_dir / "fused_data.csv")
-    # print(fusion_engine.get_summary())
-
-    print("Data fusion configured.")
-
-    # ========================================================================
-    # STEP 4: Build Physics-Informed Graph
-    # ========================================================================
-    print("\n[STEP 4] Building physics-informed graph...")
-
-    # Initialize graph builder
-    graph_builder = HydrologicalGraphBuilder(
-        allow_disconnected=True,   # Phase 1: Allow disconnected components
-        allow_cross_basin=False    # Phase 1: No cross-basin edges
-    )
-
-    # Prepare station and edge data
-    # You'll need to create these from your data:
-    # - station_df: [station_id, basin_id, latitude, longitude, elevation, ...]
-    # - edge_df: [source, target, distance, elevation_gradient, ...]
-
-    # Example structure:
-    # import pandas as pd
-    # station_df = pd.DataFrame({
-    #     'station_id': ['S1', 'S2', 'S3'],
-    #     'basin_id': [0, 0, 1],  # 0=La Risle, 1=La Eure
-    #     'latitude': [48.5, 48.6, 49.0],
-    #     'longitude': [0.5, 0.6, 0.7],
-    #     'elevation': [100, 90, 120]
-    # })
-    #
-    # edge_df = pd.DataFrame({
-    #     'source': ['S1', 'S2'],
-    #     'target': ['S2', 'S3'],
-    #     'distance': [10.5, 15.2],  # km
-    #     'elevation_gradient': [10, -30],  # m
-    #     'basin_id': [0, 1]
-    # })
-
-    # Build graph
-    # graph_data = graph_builder.build(
-    #     station_df=station_df,
-    #     edge_df=edge_df,
-    #     basin_features=None
-    # )
-
-    # Save graph
-    # graph_builder.save_graph(graph_data, graph_dir / "hydrological_graph.pt")
-    # print(graph_builder.get_graph_statistics())
-
-    print("Graph construction configured.")
-    print("Note: Use QGIS DEM hydrology tools to extract station_edges.csv")
-
-    # ========================================================================
-    # STEP 5: Prepare Tensors for GNN Training
-    # ========================================================================
-    print("\n[STEP 5] Preparing spatio-temporal tensors...")
-
-    # Initialize tensor builder
-    tensor_builder = SpatioTemporalTensorBuilder(
-        input_window=30,              # 30 days of history
-        forecast_horizons=[1, 3, 7, 14],  # Forecast 1, 3, 7, 14 days ahead
-    )
-
-    # Build tensors
-    # X, Y = tensor_builder.build_tensors(
-    #     fused_df=fused_df,
-    #     station_ids=None  # Use all stations
-    # )
-
-    # Print tensor shapes
-    # print(tensor_builder.get_tensor_shapes(X, Y))
-
-    # Save tensors
-    # tensor_builder.save_tensors(X, Y, tensor_dir / "training_tensors.pt")
-
-    print("Tensor preparation configured.")
-
-    # ========================================================================
-    # Summary
-    # ========================================================================
-    print("\n" + "="*80)
-    print("Pipeline template complete!")
-    print("="*80)
-    print("\nNext steps:")
-    print("1. Update data paths to your actual hydrometric/SAFRAN data")
-    print("2. Use QGIS to extract river network topology (station_edges.csv)")
-    print("3. Run this pipeline to generate fused data, graphs, and tensors")
-    print("4. Proceed to GNN model development (Phase 1 continuation)")
-    print("\nOutputs will be saved to:")
-    print(f"  - Fused data: {data_dir}")
-    print(f"  - Graphs: {graph_dir}")
-    print(f"  - Tensors: {tensor_dir}")
-
-
-if __name__ == "__main__":
-    main()

examples/explore_datasets.py

@@ -1,196 +0,0 @@
-"""
-Dataset exploration script.
-Efficiently explores available datasets and generates summary report.
-"""
-from pathlib import Path
-import pandas as pd
-from src.config.settings import (
-    STATION_LIST, IDPR_FILE, ADES_STATIONS, ADES_LEVELS,
-    WATERSHED_RISLE, WATERSHED_EURE, WATERSHED_MERGED
-)
-from src.utils.station_utils import load_station_metadata, get_station_summary
-from src.data.loaders.shapefile import ShapefileLoader
-from src.data.loaders.idpr import IDPRLoader
-
-
-def main():
-    """Explore all available datasets."""
-
-    print("="*80)
-    print("DATASET EXPLORATION - Hydrological GNN Project")
-    print("="*80)
-
-    # ========================================================================
-    # 1. Station Metadata
-    # ========================================================================
-    print("\n[1] STATION METADATA")
-    print("-" * 80)
-
-    try:
-        station_df = load_station_metadata(add_basin_id=True)
-        print(f"✓ Loaded {len(station_df)} stations from station_list.csv")
-        print(f"\nColumns: {list(station_df.columns)}")
-
-        # Basin summary
-        summary = get_station_summary()
-        print(f"\nBasin Distribution:")
-        print(f"  - La Risle: {summary['risle_stations']} stations")
-        print(f"  - La Eure:  {summary['eure_stations']} stations")
-
-        # Coordinate bounds
-        print(f"\nSpatial Extent:")
-        print(f"  - Longitude: {summary['bbox']['lon_min']:.3f} to {summary['bbox']['lon_max']:.3f}")
-        print(f"  - Latitude:  {summary['bbox']['lat_min']:.3f} to {summary['bbox']['lat_max']:.3f}")
-
-        # Sample stations
-        print(f"\nSample Stations (La Risle):")
-        risle_sample = station_df[station_df['basin_id'] == 0].head(3)
-        for _, row in risle_sample.iterrows():
-            print(f"  - {row['station_code']}: {row['station_name']}")
-
-        print(f"\nSample Stations (La Eure):")
-        eure_sample = station_df[station_df['basin_id'] == 1].head(3)
-        for _, row in eure_sample.iterrows():
-            print(f"  - {row['station_code']}: {row['station_name']}")
-
-    except Exception as e:
-        print(f"✗ Error loading station metadata: {e}")
-
-    # ========================================================================
-    # 2. IDPR Data
-    # ========================================================================
-    print("\n[2] IDPR DATA (Infiltration vs Runoff)")
-    print("-" * 80)
-
-    try:
-        idpr_loader = IDPRLoader(IDPR_FILE)
-        idpr_df = idpr_loader.load()
-        print(f"✓ Loaded {len(idpr_df)} IDPR values")
-        print(f"\nColumns: {list(idpr_df.columns)}")
-
-        print(f"\nIDPR Statistics:")
-        print(f"  - Mean IDPR: {idpr_df['IDPR'].mean():.1f}")
-        print(f"  - Min IDPR:  {idpr_df['IDPR'].min():.1f} (more runoff)")
-        print(f"  - Max IDPR:  {idpr_df['IDPR'].max():.1f} (more infiltration)")
-
-        # IDPR by basin
-        if 'basin_name' in idpr_df.columns:
-            print(f"\nIDPR by Basin:")
-            for basin in idpr_df['basin_name'].unique():
-                basin_data = idpr_df[idpr_df['basin_name'] == basin]
-                print(f"  - {basin}: mean={basin_data['IDPR'].mean():.1f}, "
-                      f"min={basin_data['IDPR'].min():.1f}, "
-                      f"max={basin_data['IDPR'].max():.1f}")
-
-    except Exception as e:
-        print(f"✗ Error loading IDPR data: {e}")
-
-    # ========================================================================
-    # 3. ADES Groundwater Data
-    # ========================================================================
-    print("\n[3] ADES GROUNDWATER DATA")
-    print("-" * 80)
-
-    try:
-        # Groundwater stations
-        gw_stations = pd.read_csv(ADES_STATIONS)
-        print(f"✓ Loaded {len(gw_stations)} groundwater monitoring wells")
-        print(f"\nColumns: {list(gw_stations.columns)[:10]}... ({len(gw_stations.columns)} total)")
-
-        print(f"\nGroundwater Wells Statistics:")
-        print(f"  - Mean altitude: {gw_stations['altitude_station'].mean():.1f} m")
-        print(f"  - Mean depth: {gw_stations['profondeur_investigation'].mean():.1f} m")
-        print(f"  - Total measurements: {gw_stations['nb_mesures_piezo'].sum():.0f}")
-
-        # Groundwater levels (large file - just peek)
-        print(f"\n✓ Groundwater levels file exists: {ADES_LEVELS.exists()}")
-        if ADES_LEVELS.exists():
-            # Just read first few rows
-            gw_levels_sample = pd.read_csv(ADES_LEVELS, nrows=5)
-            print(f"  - Estimated total rows: ~272,379 (large file!)")
-            print(f"  - Columns: {list(gw_levels_sample.columns)[:8]}...")
-
-    except Exception as e:
-        print(f"✗ Error loading ADES data: {e}")
-
-    # ========================================================================
-    # 4. Watershed Shapefiles
-    # ========================================================================
-    print("\n[4] WATERSHED SHAPEFILES")
-    print("-" * 80)
-
-    try:
-        # La Risle
-        risle_loader = ShapefileLoader(WATERSHED_RISLE)
-        risle_gdf = risle_loader.load()
-        print(f"✓ La Risle watershed loaded")
-        print(f"  - CRS: {risle_gdf.crs}")
-        print(f"  - Area: {risle_gdf.geometry.area.sum() / 1e6:.2f} km²")
-        print(f"  - Attributes: {list(risle_gdf.columns)}")
-
-        # La Eure
-        eure_loader = ShapefileLoader(WATERSHED_EURE)
-        eure_gdf = eure_loader.load()
-        print(f"\n✓ La Eure watershed loaded")
-        print(f"  - CRS: {eure_gdf.crs}")
-        print(f"  - Area: {eure_gdf.geometry.area.sum() / 1e6:.2f} km²")
-
-        # Merged
-        merged_loader = ShapefileLoader(WATERSHED_MERGED)
-        merged_gdf = merged_loader.load()
-        print(f"\n✓ Merged watershed loaded")
-        print(f"  - Total area: {merged_gdf.geometry.area.sum() / 1e6:.2f} km²")
-
-    except Exception as e:
-        print(f"✗ Error loading shapefiles: {e}")
-
-    # ========================================================================
-    # 5. Missing Data Summary
-    # ========================================================================
-    print("\n[5] MISSING DATA")
-    print("-" * 80)
-
-    missing_data = [
-        ("Hydrometric data (Hub'Eau)", "datasets/hydrometric/", "*.csv or *.json"),
-        ("SAFRAN meteorological data", "datasets/safran/", "*.csv or *.json"),
-        ("DEM elevation data", "datasets/DEM/", "*.tif"),
-        ("River network topology", "datasets/station_edges.csv", "CSV file"),
-        ("Basin attributes (extracted)", "datasets/basin_features/", "*.csv")
-    ]
-
-    print("The following datasets are needed to run the full pipeline:\n")
-    for name, path, format_type in missing_data:
-        print(f"  ✗ {name}")
-        print(f"    Expected: {path} ({format_type})")
-
-    # ========================================================================
-    # Summary
-    # ========================================================================
-    print("\n" + "="*80)
-    print("SUMMARY")
-    print("="*80)
-    print("\n✓ Available Data:")
-    print("  - 27 hydrometric station locations (La Risle: 12, La Eure: 13)")
-    print("  - 27 IDPR values (karst infiltration/runoff index)")
-    print("  - 114 groundwater monitoring wells (ADES)")
-    print("  - 272,379 groundwater level measurements")
-    print("  - 3 watershed boundary shapefiles (Risle, Eure, Merged)")
-
-    print("\n✗ Missing Data:")
-    print("  - Hydrometric time series (QmnJ, QIXnJ, HIXnJ) - Hub'Eau API")
-    print("  - SAFRAN meteorological time series (12 variables) - Météo-France")
-    print("  - DEM for topological feature extraction")
-    print("  - River network topology (station_edges.csv)")
-
-    print("\n📋 Next Steps:")
-    print("  1. Download hydrometric data from Hub'Eau API")
-    print("  2. Download SAFRAN meteorological data for station coordinates")
-    print("  3. Acquire DEM (IGN RGE ALTI or EU-DEM)")
-    print("  4. Use QGIS to extract river network topology")
-    print("  5. Run example_pipeline.py to process all data")
-
-    print("\n" + "="*80)
-
-
-if __name__ == "__main__":
-    main()

examples/test_fusion.py

@@ -1,50 +0,0 @@
-"""
-Test updated DataFusionEngine with actual downloaded data.
-"""
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.fusion.fusion_engine import DataFusionEngine
-
-
-def main():
-    # Initialize fusion engine
-    fusion = DataFusionEngine(
-        hydrometric_dir="datasets/hydrometric",
-        era5_dir="datasets/safran",
-        station_list_path="datasets/station_list.csv",
-        normalize=False,  # Keep raw values for inspection
-        normalization_method="robust"
-    )
-
-    # Run fusion for January 2020 (our ERA5 sample data)
-    fused_df = fusion.fuse(
-        start_date="2000-01-01",
-        end_date="2026-01-31"
-    )
-
-    # Display results
-    print("\n=== FUSED DATA SAMPLE ===")
-    print(fused_df.head(10))
-
-    print("\n=== COLUMNS ===")
-    print(fused_df.columns.tolist())
-
-    print("\n=== DATA TYPES ===")
-    print(fused_df.dtypes)
-
-    print("\n=== SUMMARY STATISTICS BY STATION ===")
-    summary = fusion.get_summary_stats()
-    print(summary)
-
-    # Save output
-    output_path = "datasets/fused/fused.csv"
-    fusion.save(output_path)
-
-    print(f"\n✓ Test complete!")
-
-
-if __name__ == "__main__":
-    main()

examples/test_graph.py

@@ -1,148 +0,0 @@
-"""
-Comprehensive test script for physics-informed hydrological graph construction and spatiotemporal dataset creation.
-"""
-import sys
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import torch
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.fusion.fusion_engine import DataFusionEngine
-from src.graph.builder import HydrologicalGraphBuilder
-from src.graph.graph_from_fused import FusedDataGraphBuilder
-from src.graph.visualize import GraphVisualizer
-
-
-def create_synthetic_fused_data():
-    """Create synthetic fused data if datasets/fused/fused.csv does not exist."""
-    print("   Creating synthetic fused data for testing...")
-    dates = pd.date_range("2020-01-01", "2020-01-31", freq="D")
-    stations = [
-        {"station_code": "H4012010", "basin_id": 0, "lat": 49.123, "lon": 0.543},
-        {"station_code": "H4023020", "basin_id": 0, "lat": 49.200, "lon": 0.600},
-        {"station_code": "H4050001", "basin_id": 0, "lat": 49.300, "lon": 0.700},
-        {"station_code": "H9011010", "basin_id": 1, "lat": 48.800, "lon": 1.200},
-        {"station_code": "H9022020", "basin_id": 1, "lat": 48.900, "lon": 1.300},
-    ]
-
-    records = []
-    for d in dates:
-        for st in stations:
-            records.append({
-                "date": d,
-                "station_code": st["station_code"],
-                "basin_id": st["basin_id"],
-                "discharge_m3s": np.random.uniform(2.0, 50.0),
-                "waterlevel_mm": np.random.uniform(300.0, 1500.0),
-                "temp_C": np.random.uniform(5.0, 18.0),
-                "wind_speed_ms": np.random.uniform(1.0, 8.0),
-                "IDPR": 120.0 if st["basin_id"] == 0 else 150.0
-            })
-
-    df = pd.DataFrame(records)
-    station_df = pd.DataFrame(stations)
-    return df, station_df
-
-
-def main():
-    print("=" * 80)
-    print("SPATIOTEMPORAL HYDROLOGICAL GRAPH CONSTRUCTION TEST")
-    print("=" * 80)
-
-    # Step 1: Load or create fused data & station metadata
-    print("\n1. Loading fused data & station list...")
-    fused_path = Path("datasets/fused/fused.csv")
-    station_path = Path("datasets/station_list.csv")
-
-    if fused_path.exists() and station_path.exists():
-        fused_df = pd.read_csv(fused_path)
-        fused_df['date'] = pd.to_datetime(fused_df['date'])
-        stations = pd.read_csv(station_path)
-        print(f"   ✓ Loaded fused data from {fused_path}")
-    else:
-        fused_df, stations = create_synthetic_fused_data()
-        print("   ✓ Synthetic test data generated.")
-
-    fused_stations = fused_df['station_code'].unique()
-    station_coords = stations[stations['station_code'].isin(fused_stations)].copy()
-    if 'station_id' not in station_coords.columns:
-        station_coords['station_id'] = station_coords['station_code']
-
-    if 'basin_id' not in station_coords.columns:
-        basin_map = fused_df[['station_code', 'basin_id']].drop_duplicates().set_index('station_code')['basin_id']
-        station_coords['basin_id'] = station_coords['station_code'].map(basin_map)
-
-    print(f"   ✓ Total observations: {len(fused_df)}")
-    print(f"   ✓ Active stations: {len(station_coords)}")
-
-    # Step 2: Test HydrologicalGraphBuilder (Physics-informed static topology)
-    print("\n2. Testing HydrologicalGraphBuilder (Static River Graph)...")
-    phy_builder = HydrologicalGraphBuilder(allow_disconnected=True, allow_cross_basin=False)
-
-    sample_edges = pd.DataFrame([
-        {"source": station_coords['station_id'].iloc[0], "target": station_coords['station_id'].iloc[1], "distance": 12.5, "elevation_gradient": 15.0, "basin_id": 0}
-    ]) if len(station_coords) >= 2 else pd.DataFrame()
-
-    static_graph = phy_builder.build(station_df=station_coords, edge_df=sample_edges)
-    stats = phy_builder.get_graph_statistics()
-
-    print(f"   ✓ Static nodes tensor shape: {static_graph.x.shape}")
-    print(f"   ✓ Static edge index shape: {static_graph.edge_index.shape}")
-    print(f"   ✓ Basin IDs tensor: {static_graph.basin_id}")
-
-    # Step 3: Test FusedDataGraphBuilder (Spatial-Temporal Snapshots & Sequences)
-    print("\n3. Testing FusedDataGraphBuilder...")
-    graph_builder = FusedDataGraphBuilder(
-        station_coords=station_coords,
-        max_distance_km=100.0,
-        allow_cross_basin=False,
-        custom_edge_index=static_graph.edge_index,
-        custom_edge_attr=static_graph.edge_attr
-    )
-
-    feature_cols = [c for c in ['discharge_m3s', 'waterlevel_mm', 'temp_C', 'wind_speed_ms', 'IDPR'] if c in fused_df.columns]
-    sample_date = fused_df['date'].iloc[0].strftime('%Y-%m-%d')
-
-    snapshot = graph_builder.build_snapshot_graph(
-        fused_df=fused_df,
-        date=sample_date,
-        feature_cols=feature_cols
-    )
-
-    print(f"   ✓ Snapshot for {sample_date}:")
-    print(f"      Node features shape: {snapshot.x.shape}")
-    print(f"      Edge index shape: {snapshot.edge_index.shape}")
-
-    # Step 4: Test Spatiotemporal Tensor Generation X in R^(N x T x S x F)
-    print("\n4. Generating 4D Spatiotemporal Tensors for GNN Training...")
-    window_size = 5
-    X_tensor, Y_tensor = graph_builder.build_spatiotemporal_tensors(
-        fused_df=fused_df,
-        window_size=window_size,
-        stride=1,
-        feature_cols=feature_cols,
-        target_cols=['discharge_m3s'] if 'discharge_m3s' in feature_cols else None
-    )
-
-    print(f"   ✓ Input Tensor X shape: {X_tensor.shape}  [N_samples, T_window, S_nodes, F_features]")
-    if Y_tensor is not None:
-        print(f"   ✓ Target Tensor Y shape: {Y_tensor.shape} [N_samples, S_nodes, Target_features]")
-
-    # Step 5: Test Visualization
-    print("\n5. Testing Graph Visualizer...")
-    try:
-        vis = GraphVisualizer(station_coords=station_coords, edge_index=snapshot.edge_index.numpy())
-        fig = vis.plot_spatial_graph(title="Test Spatial Hydrological Network")
-        print("   ✓ Spatial graph plot generated successfully.")
-    except Exception as e:
-        print(f"   ! Visualization notice: {e}")
-
-    print("\n" + "=" * 80)
-    print("ALL GRAPH PIPELINE TESTS PASSED")
-    print("=" * 80)
-
-
-if __name__ == "__main__":
-    main()

examples/test_loaders.py

@@ -1,81 +0,0 @@
-"""
-Test updated loaders with actual downloaded data.
-"""
-import sys
-from pathlib import Path
-import pandas as pd
-
-# Add src to path
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.data.loaders.hydrometric import HydrometricLoader
-from src.data.loaders.safran import SAFRANLoader
-
-
-def main():
-    print("=" * 80)
-    print("TESTING DATA LOADERS WITH ACTUAL DATA")
-    print("=" * 80)
-
-    # Test 1: Hub'Eau Hydrometric Data
-    print("\n1. Loading Hub'Eau hydrometric data...")
-    hydro_loader = HydrometricLoader(
-        data_path=Path("datasets/hydrometric"),
-        min_quality=True
-    )
-
-    hydro_df = hydro_loader.load()
-    print(f"   ✓ Loaded {len(hydro_df):,} observations")
-    print(f"   Stations: {hydro_df['station_code'].nunique()}")
-    print(f"   Date range: {hydro_df['date'].min()} to {hydro_df['date'].max()}")
-    print(f"   Columns: {list(hydro_df.columns)}")
-    print(f"\n   Sample:")
-    print(hydro_df.head(3))
-
-    # Test 2: ERA5 Data
-    print("\n2. Loading ERA5 meteorological data...")
-
-    # Load station coordinates
-    stations = pd.read_csv("datasets/station_list.csv")
-    station_coords = stations[['station_code', 'lat', 'lon']].copy()
-
-    era5_loader = SAFRANLoader(
-        data_path=Path("datasets/safran"),
-        station_coords=station_coords,
-        interp_method="nearest"
-    )
-
-    era5_df = era5_loader.load()
-    print(f"   ✓ Loaded {len(era5_df):,} observations")
-    print(f"   Stations: {era5_df['station_code'].nunique()}")
-    print(f"   Date range: {era5_df['date'].min()} to {era5_df['date'].max()}")
-    print(f"   Columns: {list(era5_df.columns)}")
-
-    # Convert units
-    print("\n3. Converting ERA5 units (K→°C, m→mm)...")
-    era5_df = SAFRANLoader.convert_units(era5_df)
-    print(f"   ✓ Converted columns: {[c for c in era5_df.columns if c not in ['date', 'station_code']]}")
-    print(f"\n   Sample:")
-    print(era5_df.head(3))
-
-    # Test 3: Merge data
-    print("\n4. Merging hydrometric + ERA5...")
-    merged = pd.merge(
-        hydro_df,
-        era5_df,
-        on=['date', 'station_code'],
-        how='inner'
-    )
-    print(f"   ✓ Merged {len(merged):,} rows")
-    print(f"   Stations: {merged['station_code'].nunique()}")
-    print(f"   Total columns: {len(merged.columns)}")
-
-    print("\n" + "=" * 80)
-    print("✓ ALL LOADERS WORKING!")
-    print("=" * 80)
-
-    return hydro_df, era5_df, merged
-
-
-if __name__ == "__main__":
-    hydro_df, era5_df, merged_df = main()

examples/visualize_full_graph.py

@@ -1,194 +0,0 @@
-"""
-Generate and save complete visualization suite for the hydrological graph dataset.
-"""
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import torch
-import networkx as nx
-import matplotlib.pyplot as plt
-
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.graph.builder import HydrologicalGraphBuilder
-from src.graph.visualize import GraphVisualizer
-
-
-def extract_station_coords(builder, fused_df: pd.DataFrame) -> pd.DataFrame:
-    """Dynamically extract station metadata from builder attributes, graph, or fused data."""
-    # 1. Search builder instance attributes for a DataFrame
-    for attr, val in builder.__dict__.items():
-        if isinstance(val, pd.DataFrame) and len(val) > 0:
-            df = val.copy()
-            if 'station_id' in df.columns and 'station_code' not in df.columns:
-                df['station_code'] = df['station_id']
-            if 'latitude' in df.columns and 'lat' not in df.columns:
-                df['lat'] = df['latitude']
-            if 'longitude' in df.columns and 'lon' not in df.columns:
-                df['lon'] = df['longitude']
-            
-            if {'station_code', 'lat', 'lon'}.issubset(df.columns) or 'basin_id' in df.columns:
-                print(f"   ✓ Extracted station metadata from builder.{attr}")
-                return df
-
-    # 2. Extract from NetworkX graph if stored in builder
-    if hasattr(builder, 'G') and isinstance(builder.G, (nx.Graph, nx.DiGraph)):
-        rows = []
-        for n, data in builder.G.nodes(data=True):
-            pos = data.get('pos', (0.0, 0.0))
-            rows.append({
-                'station_code': data.get('station_code', f"node_{n}"),
-                'lon': pos[0],
-                'lat': pos[1],
-                'basin_id': data.get('basin_id', 0)
-            })
-        if rows:
-            print("   ✓ Extracted station metadata from builder network graph (builder.G)")
-            return pd.DataFrame(rows)
-
-    # 3. Fallback: Build station table from fused dataset unique stations
-    unique_stations = (
-        fused_df['station_code'].unique() 
-        if 'station_code' in fused_df.columns 
-        else fused_df['station_id'].unique()
-    )
-    print("   ! Generating station metadata table from fused observation stations...")
-    
-    half = len(unique_stations) // 2 + 1
-    return pd.DataFrame({
-        'station_code': unique_stations,
-        'lat': [0.0 + i * 0.05 for i in range(len(unique_stations))],
-        'lon': [36.0 + i * 0.05 for i in range(len(unique_stations))],
-        'basin_id': [0 if i < half else 1 for i in range(len(unique_stations))]
-    })
-
-
-def main():
-    print("=" * 80)
-    print("HYDROLOGICAL GRAPH FULL VISUALIZATION SUITE")
-    print("=" * 80)
-
-    # 1. Setup output directory
-    output_dir = Path("outputs/plots")
-    output_dir.mkdir(parents=True, exist_ok=True)
-    print(f"\n1. Output directory ready at: {output_dir.resolve()}")
-
-    # 2. Load fused observations data
-    print("\n2. Loading fused observations dataset...")
-    fused_csv_path = Path("datasets/fused/fused.csv")
-    fused_df = pd.read_csv(fused_csv_path)
-
-    if 'station_id' in fused_df.columns and 'station_code' not in fused_df.columns:
-        fused_df['station_code'] = fused_df['station_id']
-
-    # 3. Instantiate builder & extract station metadata + edge index
-    print("\n3. Initializing HydrologicalGraphBuilder...")
-    try:
-        builder = HydrologicalGraphBuilder()
-    except Exception:
-        builder = HydrologicalGraphBuilder(fused_df)
-
-    station_coords = extract_station_coords(builder, fused_df)
-
-    if 'basin_id' not in station_coords.columns:
-        station_coords['basin_id'] = 0
-
-    edge_index = getattr(builder, 'edge_index', None)
-    if edge_index is None and hasattr(builder, 'get_edge_index'):
-        edge_index = builder.get_edge_index()
-
-    if isinstance(edge_index, torch.Tensor):
-        edge_index = edge_index.cpu().numpy()
-
-    # 4. Initialize Visualizer
-    print("\n4. Initializing GraphVisualizer...")
-    visualizer = GraphVisualizer(
-        station_coords=station_coords,
-        edge_index=edge_index,
-        figsize=(12, 8),
-        directed=True
-    )
-
-    # -------------------------------------------------------------------------
-    # Plot 1: Spatial Graph Layout
-    # -------------------------------------------------------------------------
-    print("   [1/4] Plotting Spatial Graph Network...")
-    fig1 = visualizer.plot_spatial_graph(
-        title="Hydrological Station Graph & Basin Networks",
-        show_labels=True,
-        save_path=output_dir / "01_spatial_network.png"
-    )
-    plt.close(fig1)
-
-    # -------------------------------------------------------------------------
-    # Plot 2: Node Feature Heatmap
-    # -------------------------------------------------------------------------
-    print("   [2/4] Plotting Feature Heatmap...")
-    exclude_cols = {'basin_id', 'lat', 'lon', 'latitude', 'longitude', 'station_code', 'station_id', 'date'}
-    numeric_cols = [c for c in fused_df.select_dtypes(include=[np.number]).columns if c not in exclude_cols]
-    target_feature = numeric_cols[0] if numeric_cols else fused_df.columns[2]
-
-    # Use pivot_table with aggfunc='mean' to safely aggregate duplicate timestamps per station
-    pivoted = fused_df.pivot_table(
-        index='date', 
-        columns='station_code', 
-        values=target_feature, 
-        aggfunc='mean'
-    ).ffill().bfill().fillna(0)
-
-    feature_matrix_2d = pivoted.values.T  # (num_nodes, num_timesteps)
-
-    fig2 = visualizer.plot_feature_heatmap(
-        feature_matrix=feature_matrix_2d,
-        feature_names=[target_feature],
-        title=f"Station Feature Heatmap ({target_feature})",
-        save_path=output_dir / "02_feature_heatmap.png"
-    )
-    plt.close(fig2)
-
-    # -------------------------------------------------------------------------
-    # Plot 3: Temporal Evolution Across Stations
-    # -------------------------------------------------------------------------
-    print("   [3/4] Plotting Temporal Feature Evolution...")
-    recent_df = fused_df.tail(100 * len(station_coords))
-    
-    time_series = recent_df.pivot_table(
-        index='date', 
-        columns='station_code', 
-        values=target_feature, 
-        aggfunc='mean'
-    ).ffill().bfill().fillna(0)
-
-    temporal_data = time_series.values[:, :, np.newaxis]
-    dates = [str(d)[:10] for d in time_series.index]
-
-    fig3 = visualizer.plot_temporal_evolution(
-        temporal_data=temporal_data,
-        feature_idx=0,
-        feature_name=target_feature,
-        dates=dates,
-        title=f"Temporal Dynamics of {target_feature} (Recent Windows)",
-        save_path=output_dir / "03_temporal_evolution.png"
-    )
-    plt.close(fig3)
-
-    # -------------------------------------------------------------------------
-    # Plot 4: Dataset & Graph Statistics Summary
-    # -------------------------------------------------------------------------
-    print("   [4/4] Plotting Graph Statistics Overview...")
-    fig4 = visualizer.plot_graph_statistics(
-        fused_df=fused_df,
-        save_path=output_dir / "04_graph_statistics.png"
-    )
-    plt.close(fig4)
-
-    print("\n" + "=" * 80)
-    print(f"SUCCESS: Saved all 4 plots to {output_dir.resolve()}/")
-    print("=" * 80)
-
-
-if __name__ == "__main__":
-    main()

examples/visualize_graph.py

@@ -1,210 +0,0 @@
-"""
-Visualize hydrological graphs and data.
-"""
-import sys
-from pathlib import Path
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.graph.visualize import GraphVisualizer
-
-
-def main():
-    print("=" * 80)
-    print("GRAPH VISUALIZATION")
-    print("=" * 80)
-
-    # Create output directory
-    output_dir = Path("outputs/visualizations")
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Load fused data
-    print("\n1. Loading data...")
-    fused_path = Path("datasets/fused/jan2020_fused.csv")
-    fused_df = pd.read_csv(fused_path)
-    fused_df['date'] = pd.to_datetime(fused_df['date'])
-    print(f"   ✓ Loaded {len(fused_df)} observations")
-
-    # Prepare station coordinates
-    stations = pd.read_csv("datasets/station_list.csv")
-    fused_stations = fused_df['station_code'].unique()
-    station_coords = stations[stations['station_code'].isin(fused_stations)].copy()
-    station_coords = station_coords[['station_code', 'lat', 'lon']].copy()
-
-    # Add basin_id
-    basin_map = fused_df[['station_code', 'basin_id']].drop_duplicates().set_index('station_code')['basin_id']
-    station_coords['basin_id'] = station_coords['station_code'].map(basin_map)
-    station_coords = station_coords.reset_index(drop=True)
-
-    # Build edge index (simple proximity) - compute directly to avoid torch/numpy issues
-    print("\n2. Building graph structure...")
-
-    def haversine_distance(lat1, lon1, lat2, lon2):
-        """Calculate distance between two points (km)."""
-        R = 6371  # Earth radius in km
-        lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
-        dlat = lat2 - lat1
-        dlon = lon2 - lon1
-        a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2
-        c = 2 * np.arcsin(np.sqrt(a))
-        return R * c
-
-    # Compute proximity edges
-    edges = []
-    max_distance_km = 100.0
-
-    for i, station_i in station_coords.iterrows():
-        for j, station_j in station_coords.iterrows():
-            if i >= j:
-                continue
-
-            # Check basin constraint
-            if station_i['basin_id'] != station_j['basin_id']:
-                continue
-
-            # Calculate distance
-            dist = haversine_distance(
-                station_i['lat'], station_i['lon'],
-                station_j['lat'], station_j['lon']
-            )
-
-            if dist <= max_distance_km:
-                edges.append([i, j])
-                edges.append([j, i])  # Undirected
-
-    if not edges:
-        edges = [[i, i] for i in range(len(station_coords))]
-
-    edge_index = np.array(edges).T
-    print(f"   ✓ {len(station_coords)} nodes, {edge_index.shape[1]} edges")
-
-    # Initialize visualizer
-    print("\n3. Initializing visualizer...")
-    viz = GraphVisualizer(
-        station_coords=station_coords,
-        edge_index=edge_index,
-        figsize=(12, 8)
-    )
-
-    # Plot 1: Spatial graph structure
-    print("\n4. Plotting spatial graph...")
-    fig1 = viz.plot_spatial_graph(
-        title="Hydrological Network - Geographic Layout",
-        show_labels=True,
-        save_path=output_dir / "01_spatial_graph.png"
-    )
-    plt.close(fig1)
-
-    # Plot 2: Spatial graph colored by discharge
-    print("\n5. Plotting graph with discharge values...")
-    # Get average discharge per station
-    avg_discharge = fused_df.groupby('station_code')['discharge_m3s'].mean()
-    discharge_values = station_coords['station_code'].map(avg_discharge).values
-
-    fig2 = viz.plot_spatial_graph(
-        node_values=discharge_values,
-        value_label="Avg Discharge (m³/s)",
-        title="Network Colored by Average Discharge",
-        show_labels=True,
-        save_path=output_dir / "02_discharge_spatial.png"
-    )
-    plt.close(fig2)
-
-    # Plot 3: Feature snapshot for a specific date
-    print("\n6. Plotting feature heatmap...")
-    date = "2020-01-15"
-    snapshot = fused_df[fused_df['date'] == date].copy()
-    snapshot = snapshot.set_index('station_code').reindex(station_coords['station_code'])
-
-    feature_cols = ['discharge_m3s', 'waterlevel_mm', 'temp_C', 'wind_speed_ms',
-                    'avg_groundwater_level_m', 'avg_groundwater_depth_m', 'IDPR']
-    available = [c for c in feature_cols if c in snapshot.columns]
-    feature_matrix = snapshot[available].fillna(0).values
-
-    fig3 = viz.plot_feature_heatmap(
-        feature_matrix=feature_matrix,
-        feature_names=available,
-        title=f"Node Features - {date}",
-        save_path=output_dir / "03_feature_snapshot.png"
-    )
-    plt.close(fig3)
-
-    # Plot 4: Temporal feature evolution
-    print("\n7. Plotting temporal evolution...")
-    # Prepare temporal data
-    dates = sorted(fused_df['date'].unique())[:14]  # First 2 weeks
-    temporal_data = []
-
-    for date in dates:
-        snapshot = fused_df[fused_df['date'] == date].copy()
-        snapshot = snapshot.set_index('station_code').reindex(station_coords['station_code'])
-        features = snapshot[available].fillna(0).values
-        temporal_data.append(features)
-
-    temporal_data = np.array(temporal_data)  # (time, nodes, features)
-
-    # Plot discharge evolution
-    if 'discharge_m3s' in available:
-        feat_idx = available.index('discharge_m3s')
-        fig4 = viz.plot_temporal_evolution(
-            temporal_data=temporal_data,
-            feature_idx=feat_idx,
-            feature_name='Discharge (m³/s)',
-            dates=[d.strftime('%m-%d') for d in dates],
-            title='Discharge Evolution - First 2 Weeks',
-            save_path=output_dir / "04_discharge_evolution.png"
-        )
-        plt.close(fig4)
-
-    # Plot temperature evolution
-    if 'temp_C' in available:
-        feat_idx = available.index('temp_C')
-        fig5 = viz.plot_temporal_evolution(
-            temporal_data=temporal_data,
-            feature_idx=feat_idx,
-            feature_name='Temperature (°C)',
-            dates=[d.strftime('%m-%d') for d in dates],
-            title='Temperature Evolution - First 2 Weeks',
-            save_path=output_dir / "05_temperature_evolution.png"
-        )
-        plt.close(fig5)
-
-    # Plot 5: Complete heatmap over time
-    print("\n8. Plotting temporal heatmap...")
-    fig6 = viz.plot_feature_heatmap(
-        feature_matrix=temporal_data,
-        feature_names=available,
-        dates=[d.strftime('%m-%d') for d in dates],
-        title='Features Over Time (First 2 Weeks)',
-        save_path=output_dir / "06_temporal_heatmap.png"
-    )
-    plt.close(fig6)
-
-    # Plot 6: Graph statistics overview
-    print("\n9. Plotting graph statistics...")
-    fig7 = viz.plot_graph_statistics(
-        fused_df=fused_df,
-        save_path=output_dir / "07_graph_statistics.png"
-    )
-    plt.close(fig7)
-
-    # Summary
-    print("\n" + "=" * 80)
-    print("VISUALIZATION COMPLETE")
-    print("=" * 80)
-    print(f"\n✓ Generated 7 visualizations in {output_dir}/")
-    print("\nGenerated plots:")
-    print("  01_spatial_graph.png         - Geographic network layout")
-    print("  02_discharge_spatial.png     - Network colored by discharge")
-    print("  03_feature_snapshot.png      - Feature values on 2020-01-15")
-    print("  04_discharge_evolution.png   - Discharge time series")
-    print("  05_temperature_evolution.png - Temperature time series")
-    print("  06_temporal_heatmap.png      - All features over time")
-    print("  07_graph_statistics.png      - Data overview statistics")
-
-
-if __name__ == "__main__":
-    main()