Migrate model (HF commit 5fd8672) to pvnet version 5.3.0.post0+git.489723d6.dirty

README.md

@@ -3,63 +3,73 @@ language: en
 library_name: pytorch
 license: mit
 ---
-<!--
-Do not remove elements like the above surrounded by two curly braces and do not add any more of them. These entries are required by the library and are automaticall infilled when the model is uploaded to huggingface
--->
 
-<!-- Title - e.g. PVNet-summation -->
-# TEMPLATE
 
-<!-- Provide a longer summary of what this model is/does. -->
+
+
+
+
+# PVNet2
+
 ## Model Description
 
-<!-- e.g.
-This model uses the output predictions of PVNet to predict the sum from predictions of the parts
--->
+<!-- Provide a longer summary of what this model is/does. -->
+This model class uses satellite data, numerical weather predictions, and recent Grid Service Point( GSP) PV power output to forecast the near-term (~8 hours) PV power output at all GSPs. More information can be found in the model repo [1] and experimental notes in [this google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing).
 
 - **Developed by:** openclimatefix
+- **Model type:** Fusion model
 - **Language(s) (NLP):** en
 - **License:** mit
 
+
 # Training Details
 
 ## Data
 
-<!-- eg.
-The model is trained on data from 2019-2022 and validated on data from 2022-2023. It uses the
-output predictions from PVNet - see the PVNet model for its inputs
+<!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+The model is trained on data from 2019-2022 and validated on data from 2022-2023. See experimental notes in the [the google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing) for more details.
 
--->
 
-<!-- The preprocessing section is not strictly nessessary but perhaps nice to have -->
 ### Preprocessing
 
-<!-- eg.
 Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Dataset [2].
--->
+
 
 ## Results
 
-<!-- Do not remove the lines below -->
-The training logs for this model commit can be found here:
- - [https://wandb.ai/openclimatefix/NL-Solar/runs/26kn31z5](https://wandb.ai/openclimatefix/NL-Solar/runs/26kn31z5)
+The training logs for the current model can be found here:
+ - [https://wandb.ai/openclimatefix/NL-Solar/runs/l06ylklb](https://wandb.ai/openclimatefix/NL-Solar/runs/l06ylklb)
+
+
+The training logs for all model runs of PVNet2 can be found [here](https://wandb.ai/openclimatefix/pvnet2.1).
+
+Some experimental notes can be found at in [the google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing)
 
 
-<!-- The hardware section is also just nice to have -->
 ### Hardware
-<!-- e.g.
+
 Trained on a single NVIDIA Tesla T4
--->
 
-<!-- Do not remove the section below -->
 ### Software
 
 This model was trained using the following Open Climate Fix packages:
 
-- [1] https://github.com/openclimatefix/pvnet-summation
+- [1] https://github.com/openclimatefix/PVNet
 - [2] https://github.com/openclimatefix/ocf-data-sampler
 
-<!-- Especially do not change the two lines below -->
 The versions of these packages can be found below:
- - pvnet_summation==1.1.0.post0+git.3b6f1115.dirty
- - ocf-data-sampler==0.6.0.post0+git.a508745b.dirty
+ - pvnet==4.1.18
+ - ocf-data-sampler==0.2.34
+
+
+---
+**Migration Note**: This model (HF commit 246b434b72c68edbbc13f1e5fc74e7699034aede) was migrated on 2025-09-10 to pvnet version 5.0.6.post0+git.e31d0340.dirty
+
+
+---
+**Migration Note**: This model was migrated on 2025-10-15 to pvnet version 5.1.0.post0+git.c15f1d4e.dirty
+
+
+---
+**Migration Note**: This model was migrated on 2025-12-05 to pvnet version 5.3.0.post0+git.489723d6.dirty

data_config.yaml

@@ -1,24 +1,9 @@
 general:
-  description: Initial config for NL model
-  name: Netherlands_pvnet_config
+  description: Config for training the saved PVNet model
+  name: PVNet current
 input_data:
-  site:
-    file_path: /home/alex/NL/NL_regional_generation_kw.nc
-    metadata_file_path: /home/alex/NL/NL_regional_metadata.csv
-    time_resolution_minutes: 15
-    interval_start_minutes: -2880
-    interval_end_minutes: 2160
-    dropout_timedeltas_minutes:
-    - -15
-    dropout_fraction: 1.0
-    dropout_value: -1
   nwp:
     ecmwf:
-      provider: ecmwf
-      zarr_path: PLACEHOLDER.zarr
-      interval_start_minutes: -120
-      interval_end_minutes: 2220
-      time_resolution_minutes: 60
       accum_channels:
       - direct_shortwave_radiation_flux_gl
       - downward_longwave_radiation_flux_gl
@@ -36,82 +21,158 @@ input_data:
       - snow_depth_gl
       - temperature_sl
       - total_precipitation_rate_gl
+      - visibility_sl
       - wind_u_component_10m
       - wind_v_component_10m
-      image_size_pixels_height: 10
-      image_size_pixels_width: 10
-      dropout_timedeltas_minutes:
-      - -360
       dropout_fraction: 1.0
+      dropout_timedeltas_minutes:
+      - -180
+      image_size_pixels_height: 30
+      image_size_pixels_width: 30
+      interval_end_minutes: 2160
+      interval_start_minutes: -120
       max_staleness_minutes: null
       normalisation_constants:
-        temperature_sl:
-          mean: 283.48333740234375
-          std: 3.692270040512085
-        downward_shortwave_radiation_flux_gl:
-          mean: 11458988.0
-          std: 13025427.0
-        downward_longwave_radiation_flux_gl:
-          mean: 27187026.0
-          std: 15855867.0
         cloud_cover_high:
           mean: 0.3961029052734375
           std: 0.42244860529899597
-        cloud_cover_medium:
-          mean: 0.3288780450820923
-          std: 0.38039860129356384
         cloud_cover_low:
           mean: 0.44901806116104126
           std: 0.3791404366493225
+        cloud_cover_medium:
+          mean: 0.3288780450820923
+          std: 0.38039860129356384
         cloud_cover_total:
           mean: 0.7049227356910706
           std: 0.37487083673477173
-        visibility_sl:
-          mean: 12905302.0
-          std: 16294988.0
-        snow_depth_gl:
-          mean: 8.107526082312688e-05
-          std: 0.000913831521756947
+        diff_direct_shortwave_radiation_flux_gl:
+          mean: 469169.5
+          std: 818950.6875
+        diff_downward_longwave_radiation_flux_gl:
+          mean: 1136464.0
+          std: 131942.03125
+        diff_downward_shortwave_radiation_flux_gl:
+          mean: 420584.6875
+          std: 715366.3125
+        diff_downward_ultraviolet_radiation_flux_gl:
+          mean: 48265.4765625
+          std: 81605.25
         direct_shortwave_radiation_flux_gl:
-          mean: 12905302.0
-          std: 16294988.0
+          mean: 11458988.0
+          std: 13025427.0
+        downward_longwave_radiation_flux_gl:
+          mean: 27187026.0
+          std: 15855867.0
+        downward_shortwave_radiation_flux_gl:
+          mean: 11458988.0
+          std: 13025427.0
         downward_ultraviolet_radiation_flux_gl:
           mean: 1305651.25
           std: 1445635.25
+        snow_depth_gl:
+          mean: 8.107526082312688e-05
+          std: 0.000913831521756947
+        temperature_sl:
+          mean: 283.48333740234375
+          std: 3.692270040512085
         total_precipitation_rate_gl:
           mean: 3.108070450252853e-05
           std: 9.81039775069803e-05
-        wind_u_component_10m:
-          mean: 1.7677178382873535
-          std: 5.531515598297119
+        visibility_sl:
+          mean: 12905302.0
+          std: 16294988.0
         wind_u_component_100m:
           mean: 2.393547296524048
           std: 7.2320556640625
+        wind_u_component_10m:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
         wind_u_component_200m:
           mean: 2.7963004112243652
           std: 8.049470901489258
-        wind_v_component_10m:
-          mean: 0.985887885093689
-          std: 5.411230564117432
         wind_v_component_100m:
           mean: 1.4244288206100464
           std: 6.944501876831055
+        wind_v_component_10m:
+          mean: 0.985887885093689
+          std: 5.411230564117432
         wind_v_component_200m:
           mean: 1.6010299921035767
           std: 7.561611652374268
-        diff_downward_longwave_radiation_flux_gl:
-          mean: 1136464.0
-          std: 131942.03125
-        diff_downward_shortwave_radiation_flux_gl:
-          mean: 420584.6875
-          std: 715366.3125
-        diff_downward_ultraviolet_radiation_flux_gl:
-          mean: 48265.4765625
-          std: 81605.25
-        diff_direct_shortwave_radiation_flux_gl:
-          mean: 469169.5
-          std: 818950.6875
+      provider: ecmwf
+      time_resolution_minutes: 60
+      zarr_path: PLACEHOLDER.zarr
+  satellite:
+    channels:
+    - IR_016
+    - IR_039
+    - IR_087
+    - IR_097
+    - IR_108
+    - IR_120
+    - IR_134
+    - VIS006
+    - VIS008
+    - WV_062
+    - WV_073
+    dropout_fraction: 0.8
+    dropout_timedeltas_minutes:
+    - -5
+    - -10
+    - -15
+    - -20
+    - -25
+    - -30
+    image_size_pixels_height: 100
+    image_size_pixels_width: 100
+    interval_end_minutes: -60
+    interval_start_minutes: -90
+    normalisation_constants:
+      HRV:
+        mean: 0.09298719
+        std: 0.11405209
+      IR_016:
+        mean: 0.17594202
+        std: 0.21462157
+      IR_039:
+        mean: 0.86167645
+        std: 0.04618041
+      IR_087:
+        mean: 0.7719318
+        std: 0.06687243
+      IR_097:
+        mean: 0.8014212
+        std: 0.0468558
+      IR_108:
+        mean: 0.71254843
+        std: 0.17482725
+      IR_120:
+        mean: 0.89058584
+        std: 0.06115861
+      IR_134:
+        mean: 0.944365
+        std: 0.04492306
+      VIS006:
+        mean: 0.09633306
+        std: 0.12184761
+      VIS008:
+        mean: 0.11426069
+        std: 0.13090034
+      WV_062:
+        mean: 0.7359355
+        std: 0.16111417
+      WV_073:
+        mean: 0.62479186
+        std: 0.12924142
+    time_resolution_minutes: 5
+    zarr_path: PLACEHOLDER.zarr
   solar_position:
+    interval_end_minutes: 2160
     interval_start_minutes: -2880
+    time_resolution_minutes: 15
+  generation:
+    capacity_mode: variable
     interval_end_minutes: 2160
+    interval_start_minutes: -2880
     time_resolution_minutes: 15
+    zarr_path: /home/zak/projects/PVNet/nl_solar/pv_data/netherlands_pv_data_v2.nc

model_config.yaml

@@ -1,4 +1,4 @@
-_target_: pvnet_summation.models.horizon_dense_model.HorizonDenseModel
+_target_: pvnet.models.LateFusionModel
 output_quantiles:
 - 0.02
 - 0.1
@@ -7,26 +7,45 @@ output_quantiles:
 - 0.75
 - 0.9
 - 0.98
+nwp_encoders_dict:
+  ecmwf:
+    _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
+    _partial_: true
+    in_channels: 14
+    out_features: 64
+    number_of_conv3d_layers: 4
+    conv3d_channels: 32
+    image_size_pixels: 30
+sat_encoder:
+  _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
+  _partial_: true
+  in_channels: 11
+  out_features: 256
+  number_of_conv3d_layers: 6
+  conv3d_channels: 32
+  image_size_pixels: 100
+add_image_embedding_channel: false
 output_network:
   _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
   _partial_: true
-  fc_hidden_features: 32
-  n_res_blocks: 1
+  fc_hidden_features: 128
+  n_res_blocks: 6
   res_block_layers: 2
-  dropout_frac: 0.2
-predict_difference_from_sum: true
-use_horizon_encoding: true
-use_solar_position: true
-force_non_crossing: true
-history_minutes: 2880
+  dropout_frac: 0.0
+embedding_dim: 16
+include_sun: true
+include_time: true
 forecast_minutes: 2160
+history_minutes: 2880
 interval_minutes: 15
-num_input_locations: 12
-input_quantiles:
-- 0.02
-- 0.1
-- 0.25
-- 0.5
-- 0.75
-- 0.9
-- 0.98
+min_sat_delay_minutes: 60
+sat_history_minutes: 90
+pv_history_minutes: 2880
+pv_interval_minutes: 15
+nwp_history_minutes:
+  ecmwf: 120
+nwp_forecast_minutes:
+  ecmwf: 2160
+nwp_interval_minutes:
+  ecmwf: 60
+include_generation_history: true

model_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aa051d135bc8fa1ddbf76a8ed290ea88e8c9ae7836d0f28b89703fca025f4ce
-size 23684
+oid sha256:49ff8659676eb4759756c6d1f1c65bc4c4b9815ef91bcbf541bc2f0c706d2ccb
+size 1200429264