Migrate model (HF commit bcd7035) to pvnet version 5.2.2.post15+git.fc89eaa7.dirty

README.md

@@ -14,7 +14,7 @@ Do not remove elements like the above surrounded by two curly braces and do not
 ## Model Description
 
 <!-- Provide a longer summary of what this model is/does. -->
-This model class uses satellite data, numerical weather predictions, and recent Grid Service Point( GSP) PV power output to forecast the near-term (~8 hours) PV power output at all GSPs. More information can be found in the model repo [1] and experimental notes in [this google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing).
+This model class uses satellite data, and numerical weather predictions to forecast the near-term (~8 hours) PV power output at all GSPs. More information can be found in the model repo [1].
 
 - **Developed by:** openclimatefix
 - **Model type:** Fusion model
@@ -46,14 +46,16 @@ Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Da
 
 <!-- Do not remove the lines below -->
 The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/pvnet2.1/runs/u0bc3fi9](https://wandb.ai/openclimatefix/pvnet2.1/runs/u0bc3fi9)
+ - [https://wandb.ai/openclimatefix/pvnet2.1/runs/49nlmpdy](https://wandb.ai/openclimatefix/pvnet2.1/runs/49nlmpdy)
 
 
 <!-- The hardware section is also just nice to have -->
-### Hardware
+<!-- ### Hardware 
 
 Trained on a single NVIDIA Tesla T4
 
+-->
+
 <!-- Do not remove the section below -->
 ### Software
 
@@ -64,8 +66,8 @@ This model was trained using the following Open Climate Fix packages:
 
 <!-- Especially do not change the two lines below -->
 The versions of these packages can be found below:
- - pvnet==5.0.0.post1+git.f4f6bfed.dirty
- - ocf-data-sampler==0.3.1
+ - pvnet==5.0.6.post1+git.f02c06e6.dirty
+ - ocf-data-sampler==0.5.26.post2+git.90ee263d.dirty
 
 
 ---

data_config.yaml

@@ -1,75 +1,23 @@
 general:
-  description: Config for producing batches for training PVNet+cloudcasting
+  description: na
 input_data:
+  solar_position:
+    interval_start_minutes: -120
+    interval_end_minutes: 480
+    time_resolution_minutes: 30
   nwp:
-    ecmwf:
-      accum_channels:
-      - dswrf
-      - dlwrf
-      - sr
-      - duvrs
-      channels:
-      - t2m
-      - dswrf
-      - dlwrf
-      - hcc
-      - mcc
-      - lcc
-      - tcc
-      - sd
-      - sr
-      - duvrs
-      - u10
-      - v10
-      dropout_fraction: 1.0
-      dropout_timedeltas_minutes:
-      - -360
-      image_size_pixels_height: 12
-      image_size_pixels_width: 12
-      interval_end_minutes: 480
+    ukv:
+      provider: ukv
+      zarr_path: PLACEHOLDER.zarr
       interval_start_minutes: -120
-      max_staleness_minutes: null
-      normalisation_constants:
-        diff_dlwrf:
-          mean: 1136464.0
-          std: 131942.03125
-        diff_dswrf:
-          mean: 420584.6875
-          std: 715366.3125
-        diff_duvrs:
-          mean: 48265.4765625
-          std: 81605.25
-        diff_sr:
-          mean: 469169.5
-          std: 818950.6875
-        hcc:
-          mean: 0.3961029052734375
-          std: 0.42244860529899597
-        lcc:
-          mean: 0.44901806116104126
-          std: 0.3791404366493225
-        mcc:
-          mean: 0.3288780450820923
-          std: 0.38039860129356384
-        sd:
-          mean: 8.107526082312688e-05
-          std: 0.000913831521756947
-        t2m:
-          mean: 283.48333740234375
-          std: 3.692270040512085
-        tcc:
-          mean: 0.7049227356910706
-          std: 0.37487083673477173
-        u10:
-          mean: 1.7677178382873535
-          std: 5.531515598297119
-        v10:
-          mean: 0.985887885093689
-          std: 5.411230564117432
-      provider: ecmwf
+      interval_end_minutes: 480
       time_resolution_minutes: 60
-      zarr_path: PLACEHOLDER.zarr
-    ukv:
+      image_size_pixels_height: 24
+      image_size_pixels_width: 24
+      dropout_timedeltas_minutes:
+      - -180
+      dropout_fraction: 1.0
+      max_staleness_minutes: null
       channels:
       - t
       - dswrf
@@ -82,116 +30,166 @@ input_data:
       - vis
       - si10
       - prate
-      dropout_fraction: 1.0
-      dropout_timedeltas_minutes:
-      - -180
-      image_size_pixels_height: 24
-      image_size_pixels_width: 24
-      interval_end_minutes: 480
-      interval_start_minutes: -120
-      max_staleness_minutes: null
       normalisation_constants:
-        dlwrf:
-          mean: 325.03130139
-          std: 39.45988077
+        t:
+          mean: 283.64913206
+          std: 4.38818501
         dswrf:
           mean: 111.28265039
           std: 190.47216887
+        dlwrf:
+          mean: 325.03130139
+          std: 39.45988077
         hcc:
           mean: 29.11949682
           std: 38.07184418
-        lcc:
-          mean: 50.08362643
-          std: 39.33210726
         mcc:
           mean: 40.88984494
           std: 41.91144559
-        prate:
-          mean: 3.45793433e-05
-          std: 0.00021497
-        r:
-          mean: 81.79229501
-          std: 11.45012499
+        lcc:
+          mean: 50.08362643
+          std: 39.33210726
         sde:
           mean: 0.00289545
           std: 0.1029753
-        si10:
-          mean: 6.88348448
-          std: 3.94718813
-        t:
-          mean: 283.64913206
-          std: 4.38818501
+        r:
+          mean: 81.79229501
+          std: 11.45012499
         vis:
           mean: 32262.03285118
           std: 21578.97975625
-      provider: ukv
-      time_resolution_minutes: 60
-      zarr_path: PLACEHOLDER.zarr
-    sat_pred:
-      provider: cloudcasting
+        si10:
+          mean: 6.88348448
+          std: 3.94718813
+        prate:
+          mean: 3.45793433e-05
+          std: 0.00021497
+    ecmwf:
+      provider: ecmwf
       zarr_path: PLACEHOLDER.zarr
-      interval_start_minutes: 15
-      interval_end_minutes: 180
-      time_resolution_minutes: 15
-      image_size_pixels_height: 24
-      image_size_pixels_width: 24
-      dropout_timedeltas_minutes: []
-      dropout_fraction: 0
+      interval_start_minutes: -120
+      interval_end_minutes: 480
+      time_resolution_minutes: 60
+      image_size_pixels_height: 12
+      image_size_pixels_width: 12
+      dropout_timedeltas_minutes:
+      - -360
+      dropout_fraction: 1.0
       max_staleness_minutes: null
       channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
+      - t2m
+      - dswrf
+      - dlwrf
+      - hcc
+      - mcc
+      - lcc
+      - tcc
+      - sd
+      - sr
+      - duvrs
+      - u10
+      - v10
+      accum_channels:
+      - dswrf
+      - dlwrf
+      - sr
+      - duvrs
       normalisation_constants:
-        IR_016:
-          mean: 0.17594202
-          std: 0.21462157
-        IR_039:
-          mean: 0.86167645
-          std: 0.04618041
-        IR_087:
-          mean: 0.7719318
-          std: 0.06687243
-        IR_097:
-          mean: 0.8014212
-          std: 0.0468558
-        IR_108:
-          mean: 0.71254843
-          std: 0.17482725
-        IR_120:
-          mean: 0.89058584
-          std: 0.06115861
-        IR_134:
-          mean: 0.944365
-          std: 0.04492306
-        VIS006:
-          mean: 0.09633306
-          std: 0.12184761
-        VIS008:
-          mean: 0.11426069
-          std: 0.13090034
-        WV_062:
-          mean: 0.7359355
-          std: 0.16111417
-        WV_073:
-          mean: 0.62479186
-          std: 0.12924142
-  solar_position:
-    interval_end_minutes: 480
-    interval_start_minutes: -120
-    time_resolution_minutes: 30
-  generation:
-    dropout_fraction: 0
+        t2m:
+          mean: 283.48333740234375
+          std: 3.692270040512085
+        diff_dswrf:
+          mean: 420584.6875
+          std: 715366.3125
+        diff_dlwrf:
+          mean: 1136464.0
+          std: 131942.03125
+        hcc:
+          mean: 0.3961029052734375
+          std: 0.42244860529899597
+        mcc:
+          mean: 0.3288780450820923
+          std: 0.38039860129356384
+        lcc:
+          mean: 0.44901806116104126
+          std: 0.3791404366493225
+        tcc:
+          mean: 0.7049227356910706
+          std: 0.37487083673477173
+        sd:
+          mean: 8.107526082312688e-05
+          std: 0.000913831521756947
+        diff_sr:
+          mean: 469169.5
+          std: 818950.6875
+        diff_duvrs:
+          mean: 48265.4765625
+          std: 81605.25
+        u10:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
+        v10:
+          mean: 0.985887885093689
+          std: 5.411230564117432
+  satellite:
+    zarr_path: PLACEHOLDER.zarr
+    interval_start_minutes: -60
+    interval_end_minutes: 0
+    time_resolution_minutes: 5
+    image_size_pixels_height: 24
+    image_size_pixels_width: 24
     dropout_timedeltas_minutes: []
-    interval_end_minutes: 480
+    dropout_fraction: 0.0
+    channels:
+    - IR_016
+    - IR_039
+    - IR_087
+    - IR_097
+    - IR_108
+    - IR_120
+    - IR_134
+    - VIS006
+    - VIS008
+    - WV_062
+    - WV_073
+    normalisation_constants:
+      IR_016:
+        mean: 0.17594202
+        std: 0.21462157
+      IR_039:
+        mean: 0.86167645
+        std: 0.04618041
+      IR_087:
+        mean: 0.7719318
+        std: 0.06687243
+      IR_097:
+        mean: 0.8014212
+        std: 0.0468558
+      IR_108:
+        mean: 0.71254843
+        std: 0.17482725
+      IR_120:
+        mean: 0.89058584
+        std: 0.06115861
+      IR_134:
+        mean: 0.944365
+        std: 0.04492306
+      VIS006:
+        mean: 0.09633306
+        std: 0.12184761
+      VIS008:
+        mean: 0.11426069
+        std: 0.13090034
+      WV_062:
+        mean: 0.7359355
+        std: 0.16111417
+      WV_073:
+        mean: 0.62479186
+        std: 0.12924142
+  generation:
+    zarr_path: PLACEHOLDER.zarr
     interval_start_minutes: -120
+    interval_end_minutes: 480
     time_resolution_minutes: 30
-    zarr_path: PLACEHOLDER.zarr
+    dropout_timedeltas_minutes: []
+    dropout_fraction: 0

model_config.yaml

@@ -7,7 +7,6 @@ output_quantiles:
 - 0.75
 - 0.9
 - 0.98
-add_image_embedding_channel: false
 nwp_encoders_dict:
   ukv:
     _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
@@ -25,18 +24,18 @@ nwp_encoders_dict:
     number_of_conv3d_layers: 4
     conv3d_channels: 32
     image_size_pixels: 12
-  sat_pred:
-    _target_: pvnet.models.late_fusion.encoders.encoders3d.ResConv3DNet
-    _partial_: true
-    in_channels: 11
-    out_features: 256
-    image_size_pixels: 24
-    hidden_channels: 32
-    n_res_blocks: 3
-    res_block_layers: 3
-    batch_norm: true
-    dropout_frac: 0.0
-sat_encoder: null
+sat_encoder:
+  _target_: pvnet.models.late_fusion.encoders.encoders3d.ResConv3DNet
+  _partial_: true
+  in_channels: 11
+  out_features: 256
+  image_size_pixels: 24
+  hidden_channels: 32
+  n_res_blocks: 3
+  res_block_layers: 3
+  batch_norm: true
+  dropout_frac: 0.0
+add_image_embedding_channel: false
 pv_encoder: null
 output_network:
   _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
@@ -49,18 +48,14 @@ embedding_dim: 16
 include_sun: true
 forecast_minutes: 480
 history_minutes: 120
+min_sat_delay_minutes: 0
+sat_history_minutes: 60
 nwp_history_minutes:
   ukv: 120
   ecmwf: 120
-  sat_pred: -15
 nwp_forecast_minutes:
   ukv: 480
   ecmwf: 480
-  sat_pred: 180
-nwp_interval_minutes:
-  ukv: 60
-  ecmwf: 60
-  sat_pred: 15
 location_id_mapping:
   1: 1
   2: 2

model_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e4c41f2ed9186132832d1e15cdb96062248180f9da828cff8bb837ea1678008
-size 35975432
+oid sha256:cb28ee3d8e13f28b3310054e702ba70732d67f916ee4cbcdd483ae0215ab777f
+size 36269456