Upload models - 7itrv8nl

README.md

@@ -1,54 +1,64 @@
 ---
 language: en
-license: mit
 library_name: pytorch
+license: mit
 ---
+<!--
+Do not remove elements like the above surrounded by two curly braces and do not add any more of them. These entries are required by the PVNet library and are automaticall infilled when the model is uploaded to huggingface
+-->
 
-
-
-
-
-
-# WindNet
-
-## Model Description
+<!-- Title - e.g. PVNet2, WindNet, PVNet India -->
+# WindNet with GenCast
 
 <!-- Provide a longer summary of what this model is/does. -->
-This model class uses numerical weather predictions from providers such as ECMWF to forecast the wind power in North West India over the next 48 hours at 15 minute granularity. More information can be found in the model repo [1] and experimental notes [here](https://github.com/openclimatefix/PVNet/tree/main/experiments/india).
+## Model Description
 
+<!-- e.g.
+This model uses ECMWF deterministc NWP (HRES) & GDM's GenCast forecast, it forecasts up to 48 hours ahead at 15 min intervals and is trained on regional wind power data in NW India. More information can be found in the model repo [1]. The model repo also includes links to our workshop paper on this model and some experimental notes.
+-->
 
 - **Developed by:** openclimatefix
 - **Model type:** Fusion model
 - **Language(s) (NLP):** en
 - **License:** mit
 
-
 # Training Details
 
 ## Data
 
-<!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
-
-The model is trained on data from 2019-2022 and validated on data from 2022-2023. See experimental notes [here](https://github.com/openclimatefix/PVNet/tree/main/experiments/india)
+<!-- eg.
+The model is trained on data from 2020-2022 and validated on data from 2023. This model uses ECMWF deterministc NWP (HRES) & GDM's GenCast forecast, which has its ensemble member data summarised into some aggregate stats e.g. mean, std & percentiles.
 
+See the data_config.yaml file for more information on the channels and window-size used for each input data source.
+-->
 
+<!-- The preprocessing section is not strictly nessessary but perhaps nice to have -->
 ### Preprocessing
 
-Data is prepared with the `ocf_datapipes.training.windnet` datapipe [2].
-
+<!-- eg.
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/site` Dataset [2].
+-->
 
 ## Results
 
-The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/wind_mo_global/runs/21t9xqfn](https://wandb.ai/openclimatefix/wind_mo_global/runs/21t9xqfn)
-
+<!-- Do not remove the lines below -->
+The training logs for this model commit can be found here:
+ - [https://wandb.ai/openclimatefix/gdm-gen/runs/7itrv8nl](https://wandb.ai/openclimatefix/gdm-gen/runs/7itrv8nl)
 
 
+<!-- The hardware section is also just nice to have -->
 ### Hardware
-
 Trained on a single NVIDIA Tesla T4
 
+<!-- Do not remove the section below -->
 ### Software
 
+This model was trained using the following Open Climate Fix packages:
+
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf_datapipes
+- [2] https://github.com/openclimatefix/ocf-data-sampler
+
+<!-- Especially do not change the two lines below -->
+The versions of these packages can be found below:
+ - pvnet==5.2.3
+ - ocf-data-sampler==0.6.2.post0+git.accd7422.dirty

data_config.yaml

@@ -1,71 +1,230 @@
 general:
-  description: Config for training the saved PVNet model
-  name: PVNet current
+  description: Example config for producing PVNet samples
+  name: example_config
 input_data:
-  default_forecast_minutes: 2880
-  default_history_minutes: 60
+  site:
+    time_resolution_minutes: 15
+    interval_start_minutes: 0
+    interval_end_minutes: 2880
+    file_path: /mnt/disks/gdm_data/gdm_data/ruvnl_data/india_wind_data_data_sampler_schema_v2.nc
+    metadata_file_path: /mnt/disks/gdm_data/gdm_data/ruvnl_data/india_wind_metadata_data_sampler_schema.csv
+    dropout_timedeltas_minutes: []
+    dropout_fraction: 0
   nwp:
     ecmwf:
-      coarsen_to_degrees: 0.2
-      dropout_fraction: 1.0
-      dropout_timedeltas_minutes:
-      - -360
-      forecast_minutes: 2880.0
-      history_minutes: 60
-      max_staleness_minutes: null
-      nwp_channels:
+      provider: ecmwf
+      zarr_path: PLACEHOLDER.zarr
+      interval_start_minutes: 0
+      interval_end_minutes: 2880
+      time_resolution_minutes: 60
+      channels:
       - t2m
-      - prate
       - u10
       - v10
       - u100
       - v100
       - u200
       - v200
-      nwp_image_size_pixels_height: 84
-      nwp_image_size_pixels_width: 84
-      nwp_provider: ecmwf
-      nwp_zarr_path: PLACEHOLDER.zarr
-      time_resolution_minutes: 60
-    gfs:
-      dropout_fraction: 1.0
+      image_size_pixels_height: 60
+      image_size_pixels_width: 60
       dropout_timedeltas_minutes:
-      - -300
-      forecast_minutes: 2160.0
-      history_minutes: 0
-      nwp_channels:
-      - t
-      - prate
-      - u10
-      - v10
-      - u100
-      - v100
-      nwp_image_size_pixels_height: 10
-      nwp_image_size_pixels_width: 10
-      nwp_provider: gfs
-      nwp_zarr_path: PLACEHOLDER.zarr
-      time_resolution_minutes: 180
-    mo_global:
+      - -360
       dropout_fraction: 1.0
+      max_staleness_minutes: null
+      normalisation_constants:
+        t2m:
+          mean: 283.48333740234375
+          std: 3.692270040512085
+        u10:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
+        u100:
+          mean: 2.393547296524048
+          std: 7.2320556640625
+        u200:
+          mean: 2.7963004112243652
+          std: 8.049470901489258
+        v10:
+          mean: 0.985887885093689
+          std: 5.411230564117432
+        v100:
+          mean: 1.4244288206100464
+          std: 6.944501876831055
+        v200:
+          mean: 1.6010299921035767
+          std: 7.561611652374268
+    gencast:
+      provider: gencast
+      zarr_path: PLACEHOLDER.zarr
+      interval_start_minutes: 0
+      interval_end_minutes: 2880
+      time_resolution_minutes: 360
+      channels:
+      - mean_2m_temperature
+      - mean_10m_u_component_of_wind
+      - mean_10m_v_component_of_wind
+      - mean_100m_u_component_of_wind
+      - mean_100m_v_component_of_wind
+      - std_2m_temperature
+      - std_10m_u_component_of_wind
+      - std_10m_v_component_of_wind
+      - std_100m_u_component_of_wind
+      - std_100m_v_component_of_wind
+      - P10_2m_temperature
+      - P10_10m_u_component_of_wind
+      - P10_10m_v_component_of_wind
+      - P10_100m_u_component_of_wind
+      - P10_100m_v_component_of_wind
+      - P25_2m_temperature
+      - P25_10m_u_component_of_wind
+      - P25_10m_v_component_of_wind
+      - P25_100m_u_component_of_wind
+      - P25_100m_v_component_of_wind
+      - median_2m_temperature
+      - median_10m_u_component_of_wind
+      - median_10m_v_component_of_wind
+      - median_100m_u_component_of_wind
+      - median_100m_v_component_of_wind
+      - P75_2m_temperature
+      - P75_10m_u_component_of_wind
+      - P75_10m_v_component_of_wind
+      - P75_100m_u_component_of_wind
+      - P75_100m_v_component_of_wind
+      - P90_2m_temperature
+      - P90_10m_u_component_of_wind
+      - P90_10m_v_component_of_wind
+      - P90_100m_u_component_of_wind
+      - P90_100m_v_component_of_wind
+      image_size_pixels_height: 24
+      image_size_pixels_width: 24
       dropout_timedeltas_minutes:
-      - -180
-      forecast_minutes: 1860.0
-      history_minutes: 60
+      - -480
+      dropout_fraction: 1.0
       max_staleness_minutes: null
-      nwp_channels:
-      - temperature_sl
-      - wind_u_component_10m
-      - wind_v_component_10m
-      nwp_image_size_pixels_height: 50
-      nwp_image_size_pixels_width: 50
-      nwp_provider: mo_global
-      nwp_zarr_path: PLACEHOLDER.zarr
-      time_resolution_minutes: 60
-  wind:
-    n_wind_systems_per_example: 1
-    time_resolution_minutes: 15
-    wind_files_groups:
-    - label: india
-      wind_filename: /mnt/storage_ssd_4tb/india_wind_data.nc
-      wind_metadata_filename: /mnt/storage_ssd_4tb/india_wind_metadata2.csv
-    wind_ml_ids: []
+      normalisation_constants:
+        mean_100m_u_component_of_wind:
+          mean: 0.6590617299079895
+          std: 3.5790820121765137
+        mean_100m_v_component_of_wind:
+          mean: 0.12761691212654114
+          std: 2.9323325157165527
+        mean_10m_u_component_of_wind:
+          mean: 0.4821879267692566
+          std: 2.2281882762908936
+        mean_10m_v_component_of_wind:
+          mean: 0.15181906521320343
+          std: 1.9342707395553589
+        mean_2m_temperature:
+          mean: 299.2518310546875
+          std: 7.597751617431641
+        mean_mean_sea_level_pressure:
+          mean: 100830.2890625
+          std: 680.8427124023438
+        std_100m_u_component_of_wind:
+          mean: 1.4184794425964355
+          std: 0.6928352117538452
+        std_100m_v_component_of_wind:
+          mean: 1.417962908744812
+          std: 0.6868088245391846
+        std_10m_u_component_of_wind:
+          mean: 0.9582880735397339
+          std: 0.4340810179710388
+        std_10m_v_component_of_wind:
+          mean: 0.9518918395042419
+          std: 0.44840630888938904
+        std_2m_temperature:
+          mean: 0.8567765355110168
+          std: 0.33070889115333557
+        std_mean_sea_level_pressure:
+          mean: 51.64792251586914
+          std: 32.0938835144043
+        median_100m_u_component_of_wind:
+          mean: 0.6676918268203735
+          std: 3.639376401901245
+        median_100m_v_component_of_wind:
+          mean: 0.11620805412530899
+          std: 2.985520839691162
+        median_10m_u_component_of_wind:
+          mean: 0.4807472229003906
+          std: 2.2692923545837402
+        median_10m_v_component_of_wind:
+          mean: 0.14560046792030334
+          std: 1.9702688455581665
+        median_2m_temperature:
+          mean: 299.260986328125
+          std: 7.6230292320251465
+        median_mean_sea_level_pressure:
+          mean: 100830.6796875
+          std: 680.1322021484375
+        P10_100m_u_component_of_wind:
+          mean: -1.0963581800460815
+          std: 3.7044365406036377
+        P10_100m_v_component_of_wind:
+          mean: -1.617751955986023
+          std: 3.0236403942108154
+        P10_10m_u_component_of_wind:
+          mean: -0.6955795288085938
+          std: 2.2671751976013184
+        P10_10m_v_component_of_wind:
+          mean: -1.0150115489959717
+          std: 1.971762776374817
+        P10_2m_temperature:
+          mean: 298.1833190917969
+          std: 7.59991455078125
+        P10_mean_sea_level_pressure:
+          mean: 100765.5078125
+          std: 692.2731323242188
+        P25_100m_u_component_of_wind:
+          mean: -0.2732436954975128
+          std: 3.680546283721924
+        P25_100m_v_component_of_wind:
+          mean: -0.8019947409629822
+          std: 2.9948227405548096
+        P25_10m_u_component_of_wind:
+          mean: -0.14633096754550934
+          std: 2.264906883239746
+        P25_10m_v_component_of_wind:
+          mean: -0.4697841703891754
+          std: 1.9634745121002197
+        P25_2m_temperature:
+          mean: 298.70037841796875
+          std: 7.619901657104492
+        P25_mean_sea_level_pressure:
+          mean: 100796.1953125
+          std: 685.6700439453125
+        P75_100m_u_component_of_wind:
+          mean: 1.5902334451675415
+          std: 3.6227025985717773
+        P75_100m_v_component_of_wind:
+          mean: 1.0552120208740234
+          std: 2.9857563972473145
+        P75_10m_u_component_of_wind:
+          mean: 1.1113481521606445
+          std: 2.280269145965576
+        P75_10m_v_component_of_wind:
+          mean: 0.7679195404052734
+          std: 1.9758317470550537
+        P75_2m_temperature:
+          mean: 299.8158874511719
+          std: 7.617563724517822
+        P75_mean_sea_level_pressure:
+          mean: 100865.046875
+          std: 676.5575561523438
+        P90_100m_u_component_of_wind:
+          mean: 2.4127538204193115
+          std: 3.604517936706543
+        P90_100m_v_component_of_wind:
+          mean: 1.8910019397735596
+          std: 3.0057897567749023
+        P90_10m_u_component_of_wind:
+          mean: 1.6698352098464966
+          std: 2.2776920795440674
+        P90_10m_v_component_of_wind:
+          mean: 1.3297827243804932
+          std: 1.9903264045715332
+        P90_2m_temperature:
+          mean: 300.30584716796875
+          std: 7.595254421234131
+        P90_mean_sea_level_pressure:
+          mean: 100894.1015625
+          std: 672.7207641601562

model_config.yaml

@@ -0,0 +1,56 @@
+_target_: pvnet.models.LateFusionModel
+output_quantiles:
+- 0.1
+- 0.25
+- 0.5
+- 0.75
+- 0.9
+target_key: site
+interval_minutes: 15
+embedding_dim: null
+nwp_encoders_dict:
+  ecmwf:
+    _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
+    _partial_: true
+    in_channels: 7
+    out_features: 32
+    fc_features: 32
+    number_of_conv3d_layers: 3
+    conv3d_channels: 16
+    image_size_pixels: 60
+    stride:
+    - 1
+    - 2
+    - 2
+  gencast:
+    _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
+    _partial_: true
+    in_channels: 35
+    out_features: 64
+    number_of_conv3d_layers: 8
+    fc_features: 64
+    conv3d_channels: 16
+    image_size_pixels: 24
+add_image_embedding_channel: false
+output_network:
+  _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
+  _partial_: true
+  fc_hidden_features: 32
+  n_res_blocks: 4
+  res_block_layers: 2
+  dropout_frac: 0.2
+include_sun: false
+include_time: true
+include_gsp_yield_history: false
+include_site_yield_history: false
+forecast_minutes: 2880
+history_minutes: 0
+nwp_history_minutes:
+  gencast: 0
+  ecmwf: 0
+nwp_forecast_minutes:
+  gencast: 2880
+  ecmwf: 2880
+nwp_interval_minutes:
+  gencast: 360
+  ecmwf: 60

model_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d21a0a811b0c151f6d07516f970e268d064338711d5ed650e1700a92db4903f5
+size 6605176