Upload folder using huggingface_hub

.all-contributorsrc

@@ -0,0 +1,189 @@
+{
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitType": "docs",
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "felix-e-h-p",
+      "name": "Felix",
+      "avatar_url": "https://avatars.githubusercontent.com/u/137530077?v=4",
+      "profile": "https://github.com/felix-e-h-p",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "Sukh-P",
+      "name": "Sukhil Patel",
+      "avatar_url": "https://avatars.githubusercontent.com/u/42407101?v=4",
+      "profile": "https://github.com/Sukh-P",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "dfulu",
+      "name": "James Fulton",
+      "avatar_url": "https://avatars.githubusercontent.com/u/41546094?v=4",
+      "profile": "https://github.com/dfulu",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "AUdaltsova",
+      "name": "Alexandra Udaltsova",
+      "avatar_url": "https://avatars.githubusercontent.com/u/43303448?v=4",
+      "profile": "https://github.com/AUdaltsova",
+      "contributions": [
+        "code",
+        "review"
+      ]
+    },
+    {
+      "login": "zakwatts",
+      "name": "Megawattz",
+      "avatar_url": "https://avatars.githubusercontent.com/u/47150349?v=4",
+      "profile": "https://github.com/zakwatts",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "peterdudfield",
+      "name": "Peter Dudfield",
+      "avatar_url": "https://avatars.githubusercontent.com/u/34686298?v=4",
+      "profile": "https://github.com/peterdudfield",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "mahdilamb",
+      "name": "Mahdi Lamb",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4696915?v=4",
+      "profile": "https://github.com/mahdilamb",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "jacobbieker",
+      "name": "Jacob Prince-Bieker",
+      "avatar_url": "https://avatars.githubusercontent.com/u/7170359?v=4",
+      "profile": "https://www.jacobbieker.com",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "codderrrrr",
+      "name": "codderrrrr",
+      "avatar_url": "https://avatars.githubusercontent.com/u/149995852?v=4",
+      "profile": "https://github.com/codderrrrr",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "confusedmatrix",
+      "name": "Chris Briggs",
+      "avatar_url": "https://avatars.githubusercontent.com/u/617309?v=4",
+      "profile": "https://chrisxbriggs.com",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmi",
+      "name": "tmi",
+      "avatar_url": "https://avatars.githubusercontent.com/u/147159?v=4",
+      "profile": "https://github.com/tmi",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "carderne",
+      "name": "Chris Arderne",
+      "avatar_url": "https://avatars.githubusercontent.com/u/19817302?v=4",
+      "profile": "https://rdrn.me/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "Dakshbir",
+      "name": "Dakshbir",
+      "avatar_url": "https://avatars.githubusercontent.com/u/144359831?v=4",
+      "profile": "https://github.com/Dakshbir",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "MAYANK12SHARMA",
+      "name": "MAYANK SHARMA",
+      "avatar_url": "https://avatars.githubusercontent.com/u/145884197?v=4",
+      "profile": "https://github.com/MAYANK12SHARMA",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "lambaaryan011",
+      "name": "aryan lamba ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/153702847?v=4",
+      "profile": "https://github.com/lambaaryan011",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "michael-gendy",
+      "name": "michael-gendy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/64384201?v=4",
+      "profile": "https://github.com/michael-gendy",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "adityasuthar",
+      "name": "Aditya Suthar",
+      "avatar_url": "https://avatars.githubusercontent.com/u/95685363?v=4",
+      "profile": "https://adityasuthar.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "markus-kreft",
+      "name": "Markus Kreft",
+      "avatar_url": "https://avatars.githubusercontent.com/u/129367085?v=4",
+      "profile": "https://github.com/markus-kreft",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "JackKelly",
+      "name": "Jack Kelly",
+      "avatar_url": "https://avatars.githubusercontent.com/u/460756?v=4",
+      "profile": "http://jack-kelly.com",
+      "contributions": [
+        "ideas"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true,
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "projectName": "PVNet",
+  "projectOwner": "openclimatefix"
+}

.bumpversion.cfg

@@ -0,0 +1,9 @@
+[bumpversion]
+commit = True
+tag = True
+current_version = 4.1.18
+message = Bump version: {current_version} → {new_version} [skip ci]
+
+[bumpversion:file:pvnet/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"

.gitattributes

@@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+experiments/india/003_wind_plevels/MAE.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/003_wind_plevels/MAEvstimesteps.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/003_wind_plevels/p10.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/003_wind_plevels/p50.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/004_n_training_samples/mae_step.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/005_extra_nwp_variables/mae_steps.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/005_extra_nwp_variables/mae_steps_grouped.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/006_da_only/bad.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/006_da_only/good.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/006_da_only/mae_steps.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/007_different_seeds/mae_all_steps.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/007_different_seeds/mae_steps.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/008_coarse4/mae_step.png filter=lfs diff=lfs merge=lfs -text
+experiments/india/008_coarse4/mae_step_smooth.png filter=lfs diff=lfs merge=lfs -text
+experiments/uk/011[[:space:]]-[[:space:]]Extending[[:space:]]forecast[[:space:]]to[[:space:]]36[[:space:]]hours[[:space:]](updated[[:space:]]ECMWF[[:space:]]data)/PVNEt_national_XG_comparison.png filter=lfs diff=lfs merge=lfs -text
+experiments/uk/011[[:space:]]-[[:space:]]Extending[[:space:]]forecast[[:space:]]to[[:space:]]36[[:space:]]hours[[:space:]](updated[[:space:]]ECMWF[[:space:]]data)/PVNets_comparison.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/release.yml

@@ -0,0 +1,17 @@
+name: Python Bump Version & release
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "configs.example/**" # ignores all files in configs.example
+      - "**/README.md" # ignores all README files
+      - "experiments/**" # ignores all files in experiments directory
+
+jobs:
+  release:
+    uses: openclimatefix/.github/.github/workflows/python-release.yml@main
+    secrets:
+      token: ${{ secrets.PYPI_API_TOKEN }}
+      PAT_TOKEN: ${{ secrets.PAT_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Python package tests
+
+on:
+  push:
+  pull_request:
+    types: [opened, reopened]
+  schedule:
+    - cron: "0 12 * * 1"
+jobs:
+  call-run-python-tests:
+    uses: openclimatefix/.github/.github/workflows/python-test.yml@main
+    with:
+      # 0 means don't use pytest-xdist
+      pytest_numcpus: "4"
+      # pytest-cov looks at this folder
+      pytest_cov_dir: "pvnet"
+      # extra things to install
+      sudo_apt_install: "libgeos++-dev libproj-dev proj-data proj-bin"
+      #      brew_install: "proj geos librttopo"
+      os_list: '["ubuntu-latest"]'
+      python-version: "['3.10', '3.11']"
+      extra_commands: "pip3 install -e '.[all]'"

.gitignore

@@ -0,0 +1,146 @@
+# Custom
+config_tree.txt
+configs/
+lightning_logs/
+logs/
+output/
+checkpoints*
+csv/
+notebooks/
+*.html
+*.csv
+latest_logged_train_batch.png
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+.DS_Store
+
+# vim
+*swp

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      # list of supported hooks: https://pre-commit.com/hooks.html
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: debug-statements
+      - id: detect-private-key
+
+  # python code formatting/linting
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: "v0.11.0"
+    hooks:
+      - id: ruff
+        args: [--fix]
+  # yaml formatting
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.2
+    hooks:
+      - id: prettier
+        types: [yaml]

.prettierignore

@@ -0,0 +1 @@
+configs.example

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Open Climate Fix Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,2 @@
+include *.txt
+recursive-include pvnet/models/model_cards *.md

README.md

@@ -39,7 +39,6 @@ Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Da
 ## Results
 
 The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/NL-Solar/runs/hc2sv5fj](https://wandb.ai/openclimatefix/NL-Solar/runs/hc2sv5fj)
 
 
 The training logs for all model runs of PVNet2 can be found [here](https://wandb.ai/openclimatefix/pvnet2.1).
@@ -53,5 +52,11 @@ Trained on a single NVIDIA Tesla T4
 
 ### Software
 
+This model was trained using the following Open Climate Fix packages:
+
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf-data-sampler
+- [2] https://github.com/openclimatefix/ocf-data-sampler
+
+The versions of these packages can be found below:
+ - pvnet==4.1.18
+ - ocf-data-sampler==0.2.31

configs.example/callbacks/default.yaml

@@ -0,0 +1,30 @@
+early_stopping:
+  _target_: pvnet.callbacks.MainEarlyStopping
+  # name of the logged metric which determines when model is improving
+  monitor: "${resolve_monitor_loss:${model.output_quantiles}}"
+  mode: "min" # can be "max" or "min"
+  patience: 10 # how many epochs (or val check periods) of not improving until training stops
+  min_delta: 0 # minimum change in the monitored metric needed to qualify as an improvement
+
+learning_rate_monitor:
+  _target_: lightning.pytorch.callbacks.LearningRateMonitor
+  logging_interval: "epoch"
+
+model_summary:
+  _target_: lightning.pytorch.callbacks.ModelSummary
+  max_depth: 3
+
+model_checkpoint:
+  _target_: lightning.pytorch.callbacks.ModelCheckpoint
+  # name of the logged metric which determines when model is improving
+  monitor: "${resolve_monitor_loss:${model.output_quantiles}}"
+  mode: "min" # can be "max" or "min"
+  save_top_k: 1 # save k best models (determined by above metric)
+  save_last: True # additionaly always save model from last epoch
+  every_n_epochs: 1
+  verbose: False
+  filename: "epoch={epoch}-step={step}"
+  # The path to where the model checkpoints will be stored
+  dirpath: "PLACEHOLDER/${model_name}" #${..model_name}
+  auto_insert_metric_name: False
+  save_on_train_epoch_end: False

configs.example/callbacks/wandb.yaml

@@ -0,0 +1,26 @@
+defaults:
+  - default.yaml
+
+watch_model:
+  _target_: src.callbacks.wandb_callbacks.WatchModel
+  log: "all"
+  log_freq: 100
+
+upload_code_as_artifact:
+  _target_: src.callbacks.wandb_callbacks.UploadCodeAsArtifact
+  code_dir: ${work_dir}/src
+
+upload_ckpts_as_artifact:
+  _target_: src.callbacks.wandb_callbacks.UploadCheckpointsAsArtifact
+  ckpt_dir: "checkpoints/"
+  upload_best_only: True
+
+log_f1_precision_recall_heatmap:
+  _target_: src.callbacks.wandb_callbacks.LogF1PrecRecHeatmap
+
+log_confusion_matrix:
+  _target_: src.callbacks.wandb_callbacks.LogConfusionMatrix
+
+log_image_predictions:
+  _target_: src.callbacks.wandb_callbacks.LogImagePredictions
+  num_samples: 8

configs.example/config.yaml

@@ -0,0 +1,45 @@
+# @package _global_
+
+# specify here default training configuration
+defaults:
+  - _self_
+  - trainer: default.yaml
+  - model: multimodal.yaml
+  - datamodule: premade_samples.yaml
+  - callbacks: default.yaml # set this to null if you don't want to use callbacks
+  - logger: wandb.yaml # set logger here or use command line (e.g. `python run.py logger=wandb`)
+  - experiment: null
+  - hparams_search: null
+  - hydra: default.yaml
+
+renewable: "pv_uk"
+
+# enable color logging
+#  - override hydra/hydra_logging: colorlog
+#  - override hydra/job_logging: colorlog
+
+# path to original working directory
+# hydra hijacks working directory by changing it to the current log directory,
+# so it's useful to have this path as a special variable
+# learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory
+work_dir: ${hydra:runtime.cwd}
+
+model_name: "default"
+
+# use `python run.py debug=true` for easy debugging!
+# this will run 1 train, val and test loop with only 1 batch
+# equivalent to running `python run.py trainer.fast_dev_run=true`
+# (this is placed here just for easier access from command line)
+debug: False
+
+# pretty print config at the start of the run using Rich library
+print_config: True
+
+# disable python warnings if they annoy you
+ignore_warnings: True
+
+# check performance on test set, using the best model achieved during training
+# lightning chooses best model based on metric specified in checkpoint callback
+test_after_training: False
+
+seed: 2727831

configs.example/datamodule/configuration/example_configuration.yaml

@@ -0,0 +1,288 @@
+general:
+  description: Example config for producing PVNet samples
+  name: example_config
+
+input_data:
+
+  # Either use Site OR GSP configuration
+  site:
+    # Path to Site data in NetCDF format
+    file_path: PLACEHOLDER.nc
+    # Path to metadata in CSV format
+    metadata_file_path: PLACEHOLDER.csv
+    time_resolution_minutes: 15
+    interval_start_minutes: -60
+    # Specified for intraday currently
+    interval_end_minutes: 480
+    dropout_timedeltas_minutes: []
+    dropout_fraction: 0 # Fraction of samples with dropout
+
+  gsp:
+    # Path to GSP data in zarr format
+    # e.g. gs://solar-pv-nowcasting-data/PV/GSP/v7/pv_gsp.zarr
+    zarr_path: PLACEHOLDER.zarr
+    interval_start_minutes: -60
+    # Specified for intraday currently
+    interval_end_minutes: 480
+    time_resolution_minutes: 30
+    # Random value from the list below will be chosen as the delay when dropout is used
+    # If set to null no dropout is applied. Only values before t0 are dropped out for GSP.
+    # Values after t0 are assumed as targets and cannot be dropped.
+    dropout_timedeltas_minutes: []
+    dropout_fraction: 0 # Fraction of samples with dropout
+
+  nwp:
+
+    ecmwf:
+      provider: ecmwf
+      # Path to ECMWF NWP data in zarr format
+      # n.b. It is not necessary to use multiple or any NWP data. These entries can be removed
+      zarr_path: PLACEHOLDER.zarr
+      interval_start_minutes: -60
+      # Specified for intraday currently
+      interval_end_minutes: 480
+      time_resolution_minutes: 60
+      channels:
+        - t2m # 2-metre temperature
+        - dswrf # downwards short-wave radiation flux
+        - dlwrf # downwards long-wave radiation flux
+        - hcc # high cloud cover
+        - mcc # medium cloud cover
+        - lcc # low cloud cover
+        - tcc # total cloud cover
+        - sde # snow depth water equivalent
+        - sr # direct solar radiation
+        - duvrs # downwards UV radiation at surface
+        - prate # precipitation rate
+        - u10 # 10-metre U component of wind speed
+        - u100 # 100-metre U component of wind speed
+        - u200 # 200-metre U component of wind speed
+        - v10 # 10-metre V component of wind speed
+        - v100 # 100-metre V component of wind speed
+        - v200 # 200-metre V component of wind speed
+      # The following channels are accumulated and need to be diffed
+      accum_channels:
+        - dswrf # downwards short-wave radiation flux
+        - dlwrf # downwards long-wave radiation flux
+        - sr # direct solar radiation
+        - duvrs # downwards UV radiation at surface
+      image_size_pixels_height: 24
+      image_size_pixels_width: 24
+      dropout_timedeltas_minutes: [-360]
+      dropout_fraction: 1.0 # Fraction of samples with dropout
+      max_staleness_minutes: null
+      normalisation_constants:
+        t2m:
+          mean: 283.48333740234375
+          std: 3.692270040512085
+        dswrf:
+          mean: 11458988.0
+          std: 13025427.0
+        dlwrf:
+          mean: 27187026.0
+          std: 15855867.0
+        hcc:
+          mean: 0.3961029052734375
+          std: 0.42244860529899597
+        mcc:
+          mean: 0.3288780450820923
+          std: 0.38039860129356384
+        lcc:
+          mean: 0.44901806116104126
+          std: 0.3791404366493225
+        tcc:
+          mean: 0.7049227356910706
+          std: 0.37487083673477173
+        sde:
+          mean: 8.107526082312688e-05
+          std: 0.000913831521756947  # Mapped from "sd" in the Python file
+        sr:
+          mean: 12905302.0
+          std: 16294988.0
+        duvrs:
+          mean: 1305651.25
+          std: 1445635.25
+        prate:
+          mean: 3.108070450252853e-05
+          std: 9.81039775069803e-05
+        u10:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
+        u100:
+          mean: 2.393547296524048
+          std: 7.2320556640625
+        u200:
+          mean: 2.7963004112243652
+          std: 8.049470901489258
+        v10:
+          mean: 0.985887885093689
+          std: 5.411230564117432
+        v100:
+          mean: 1.4244288206100464
+          std: 6.944501876831055
+        v200:
+          mean: 1.6010299921035767
+          std: 7.561611652374268
+        # Added diff_ keys for the channels under accum_channels:
+        diff_dlwrf:
+          mean: 1136464.0
+          std: 131942.03125
+        diff_dswrf:
+          mean: 420584.6875
+          std: 715366.3125
+        diff_duvrs:
+          mean: 48265.4765625
+          std: 81605.25
+        diff_sr:
+          mean: 469169.5
+          std: 818950.6875
+
+    ukv:
+      provider: ukv
+      # Path to UKV NWP data in zarr format
+      # e.g. gs://solar-pv-nowcasting-data/NWP/UK_Met_Office/UKV_intermediate_version_7.zarr
+      # n.b. It is not necessary to use multiple or any NWP data. These entries can be removed
+      zarr_path: PLACEHOLDER.zarr
+      interval_start_minutes: -60
+      # Specified for intraday currently
+      interval_end_minutes: 480
+      time_resolution_minutes: 60
+      channels:
+        - t # 2-metre temperature
+        - dswrf # downwards short-wave radiation flux
+        - dlwrf # downwards long-wave radiation flux
+        - hcc # high cloud cover
+        - mcc # medium cloud cover
+        - lcc # low cloud cover
+        - sde # snow depth water equivalent
+        - r # relative humidty
+        - vis # visibility
+        - si10 # 10-metre wind speed
+        - wdir10 # 10-metre wind direction
+        - prate # precipitation rate
+        # These variables exist in CEDA training data but not in the live MetOffice live service
+        - hcct # height of convective cloud top, meters above surface. NaN if no clouds
+        - cdcb # height of lowest cloud base > 3 oktas
+        - dpt # dew point temperature
+        - prmsl # mean sea level pressure
+        - h # geometrical? (maybe geopotential?) height
+      image_size_pixels_height: 24
+      image_size_pixels_width: 24
+      dropout_timedeltas_minutes: [-360]
+      dropout_fraction: 1.0 # Fraction of samples with dropout
+      max_staleness_minutes: null
+      normalisation_constants:
+        t:
+          mean: 283.64913206
+          std: 4.38818501
+        dswrf:
+          mean: 111.28265039
+          std: 190.47216887
+        dlwrf:
+          mean: 325.03130139
+          std: 39.45988077
+        hcc:
+          mean: 29.11949682
+          std: 38.07184418
+        mcc:
+          mean: 40.88984494
+          std: 41.91144559
+        lcc:
+          mean: 50.08362643
+          std: 39.33210726
+        sde:
+          mean: 0.00289545
+          std: 0.1029753
+        r:
+          mean: 81.79229501
+          std: 11.45012499
+        vis:
+          mean: 32262.03285118
+          std: 21578.97975625
+        si10:
+          mean: 6.88348448
+          std: 3.94718813
+        wdir10:
+          mean: 199.41891636
+          std: 94.08407495
+        prate:
+          mean: 3.45793433e-05
+          std: 0.00021497
+        hcct:
+          mean: -18345.97478167
+          std: 18382.63958991
+        cdcb:
+          mean: 1412.26599062
+          std: 2126.99350113
+        dpt:
+          mean: 280.54379901
+          std: 4.57250482
+        prmsl:
+          mean: 101321.61574029
+          std: 1252.71790539
+        h:
+          mean: 2096.51991356
+          std: 1075.77812282
+
+  satellite:
+    # Path to Satellite data (non-HRV) in zarr format
+    # e.g. gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+    zarr_path: PLACEHOLDER.zarr
+    interval_start_minutes: -30
+    interval_end_minutes: 0
+    time_resolution_minutes: 5
+    channels:
+      - IR_016 # Surface, cloud phase
+      - IR_039 # Surface, clouds, wind fields
+      - IR_087 # Surface, clouds, atmospheric instability
+      - IR_097 # Ozone
+      - IR_108 # Surface, clouds, wind fields, atmospheric instability
+      - IR_120 # Surface, clouds, atmospheric instability
+      - IR_134 # Cirrus cloud height, atmospheric instability
+      - VIS006 # Surface, clouds, wind fields
+      - VIS008 # Surface, clouds, wind fields
+      - WV_062 # Water vapor, high level clouds, upper air analysis
+      - WV_073 # Water vapor, atmospheric instability, upper-level dynamics
+    image_size_pixels_height: 24
+    image_size_pixels_width: 24
+    dropout_timedeltas_minutes: []
+    dropout_fraction: 0 # Fraction of samples with dropout
+    normalisation_constants:
+      IR_016:
+        mean: 0.17594202
+        std: 0.21462157
+      IR_039:
+        mean: 0.86167645
+        std: 0.04618041
+      IR_087:
+        mean: 0.7719318
+        std: 0.06687243
+      IR_097:
+        mean: 0.8014212
+        std: 0.0468558
+      IR_108:
+        mean: 0.71254843
+        std: 0.17482725
+      IR_120:
+        mean: 0.89058584
+        std: 0.06115861
+      IR_134:
+        mean: 0.944365
+        std: 0.04492306
+      VIS006:
+        mean: 0.09633306
+        std: 0.12184761
+      VIS008:
+        mean: 0.11426069
+        std: 0.13090034
+      WV_062:
+        mean: 0.7359355
+        std: 0.16111417
+      WV_073:
+        mean: 0.62479186
+        std: 0.12924142
+
+  solar_position:
+    interval_start_minutes: -60
+    interval_end_minutes: 480
+    time_resolution_minutes: 30

configs.example/datamodule/premade_batches.yaml

@@ -0,0 +1,10 @@
+_target_: pvnet.data.DataModule
+configuration: null
+
+# The sample_dir is the location batches were saved to using the save_batches.py script
+# The sample_dir should contain train and val subdirectories with batches
+
+sample_dir: "PLACEHOLDER"
+num_workers: 10
+prefetch_factor: 2
+batch_size: 8

configs.example/datamodule/streamed_batches.yaml

@@ -0,0 +1,20 @@
+_target_: pvnet.data.DataModule
+# Path to the data configuration yaml file. You can find examples in the configuration subdirectory
+# in configs.example/datamodule/configuration
+# Use the full local path such as: /FULL/PATH/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+
+configuration: "PLACEHOLDER.yaml"
+num_workers: 20
+prefetch_factor: 2
+batch_size: 8
+
+sample_output_dir: "PLACEHOLDER"
+num_train_samples: 2
+num_val_samples: 1
+
+train_period:
+  - null
+  - "2022-05-07"
+val_period:
+  - "2022-05-08"
+  - "2023-05-08"

configs.example/experiment/baseline.yaml

@@ -0,0 +1,21 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=example_simple.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: baseline.yaml
+  - override /datamodule: premade_samples.yaml
+  - override /callbacks: default.yaml
+  - override /logger: neptune.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+validate_only: "1" # by putting this key in the config file, the model does not get trained.
+
+trainer:
+  min_epochs: 1
+  max_epochs: 1

configs.example/experiment/conv3d_sat_nwp.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=example_simple.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: conv3d_sat_nwp.yaml
+  - override /datamodule: premade_samples.yaml
+  - override /callbacks: default.yaml
+#  - override /logger: neptune.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+
+trainer:
+  min_epochs: 1
+  max_epochs: 10
+
+model:
+  conv3d_channels: 32

configs.example/experiment/example_simple.yaml

@@ -0,0 +1,27 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=example_simple.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: conv3d_sat_nwp.yaml
+  - override /datamodule: premade_samples.yaml
+  - override /callbacks: default.yaml
+  - override /logger: tensorboard.yaml
+  - override /hparams_search: null
+  - override /hydra: default.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+
+trainer:
+  min_epochs: 1
+  max_epochs: 2
+
+datamodule:
+  batch_size: 16
+
+validate_only: "1" # by putting this key in the config file, the model does not get trained.

configs.example/experiment/test.yaml

@@ -0,0 +1,33 @@
+# @package _global_
+
+# to execute this experiment run:
+# python run.py experiment=test.yaml
+
+defaults:
+  - override /trainer: default.yaml # choose trainer from 'configs/trainer/'
+  - override /model: test.yaml
+  - override /datamodule: premade_samples.yaml
+  - override /callbacks: default.yaml
+
+# all parameters below will be merged with parameters from default configurations set above
+# this allows you to overwrite only specified parameters
+
+seed: 518
+
+trainer:
+  min_epochs: 0
+  max_epochs: 2
+  reload_dataloaders_every_n_epochs: 0
+  limit_train_batches: 2000
+  limit_val_batches: 100
+  limit_test_batches: 100
+  val_check_interval: 100
+  num_sanity_val_steps: 8
+  accumulate_grad_batches: 4
+  #fast_dev_run: 3
+
+datamodule:
+  num_workers: 10
+  prefetch_factor: 2
+  batch_size: 8
+#validate_only: '1'  # by putting this key in the config file, the model does not get trained.

configs.example/hparams_search/conv3d_optuna.yaml

@@ -0,0 +1,49 @@
+# @package _global_
+
+# example hyperparameter optimization of some experiment with Optuna:
+# python run.py -m hparams_search=conv3d_optuna experiment=conv3d_sat_nwp
+
+defaults:
+  - override /hydra/sweeper: optuna
+
+# choose metric which will be optimized by Optuna
+optimized_metric: "MSE/Validation_epoch"
+
+hydra:
+  # here we define Optuna hyperparameter search
+  # it optimizes for value returned from function with @hydra.main decorator
+  # learn more here: https://hydra.cc/docs/next/plugins/optuna_sweeper
+  sweeper:
+    _target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper
+    storage: null
+    study_name: null
+    n_jobs: 1
+
+    # 'minimize' or 'maximize' the objective
+    direction: minimize
+
+    # number of experiments that will be executed
+    n_trials: 20
+
+    # choose Optuna hyperparameter sampler
+    # learn more here: https://optuna.readthedocs.io/en/stable/reference/samplers.html
+    sampler:
+      _target_: optuna.samplers.TPESampler
+      seed: 12345
+      consider_prior: true
+      prior_weight: 1.0
+      consider_magic_clip: true
+      consider_endpoints: false
+      n_startup_trials: 10
+      n_ei_candidates: 24
+      multivariate: false
+      warn_independent_sampling: true
+
+    # define range of hyperparameters
+    search_space:
+      model.include_pv_yield_history:
+        type: categorical
+        choices: [true, false]
+      model.include_future_satellite:
+        type: categorical
+        choices: [true, false]

configs.example/hydra/default.yaml

@@ -0,0 +1,14 @@
+# output paths for hydra logs
+run:
+  # Local log directory for hydra
+  dir: PLACEHOLDER/runs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+sweep:
+  # Local log directory for hydra
+  dir: PLACEHOLDER/multiruns/${now:%Y-%m-%d_%H-%M-%S}
+  subdir: ${hydra.job.num}
+
+# you can set here environment variables that are universal for all users
+# for system specific variables (like data paths) it's better to use .env file!
+job:
+  env_set:
+    EXAMPLE_VAR: "example_value"

configs.example/logger/csv.yaml

@@ -0,0 +1,9 @@
+# csv logger built in lightning
+
+csv:
+  _target_: pytorch_lightning.loggers.csv_logs.CSVLogger
+  # local path to log training process
+  save_dir: "PLACEHOLDER"
+  name: "csv/"
+  version: null
+  prefix: ""

configs.example/logger/many_loggers.yaml

@@ -0,0 +1,7 @@
+# train with many loggers at once
+
+defaults:
+  - csv.yaml
+  # - neptune.yaml
+  # - tensorboard.yaml
+  - wandb.yaml

configs.example/logger/neptune.yaml

@@ -0,0 +1,8 @@
+# https://neptune.ai
+
+neptune:
+  _target_: pytorch_lightning.loggers.NeptuneLogger
+  api_key: ${oc.env:NEPTUNE_API_TOKEN} # api key is loaded from environment variable
+  # Neptune project placeholder
+  project: PLACEHOLDER
+  prefix: ""

configs.example/logger/tensorboard.yaml

@@ -0,0 +1,11 @@
+# https://www.tensorflow.org/tensorboard/
+
+tensorboard:
+  _target_: pytorch_lightning.loggers.tensorboard.TensorBoardLogger
+  # Path to use for tensorboard logs
+  save_dir: "PLACEHOLDER"
+  name: "default"
+  version: "${model_name}"
+  log_graph: False
+  default_hp_metric: False
+  prefix: ""

configs.example/logger/wandb.yaml

@@ -0,0 +1,17 @@
+# https://wandb.ai
+
+wandb:
+  _target_: lightning.pytorch.loggers.wandb.WandbLogger
+  # wandb project to log to
+  project: "PLACEHOLDER"
+  name: "${model_name}"
+  # location to store the wandb local logs
+  save_dir: "PLACEHOLDER"
+  offline: False # set True to store all logs only locally
+  id: null # pass correct id to resume experiment!
+  # entity: ""  # set to name of your wandb team or just remove it
+  log_model: False
+  prefix: ""
+  job_type: "train"
+  group: ""
+  tags: []

configs.example/model/baseline.yaml

@@ -0,0 +1,4 @@
+_target_: pvnet.models.baseline.last_value.Model
+
+forecast_minutes: 120
+history_minutes: 30

configs.example/model/multimodal.yaml

@@ -0,0 +1,115 @@
+_target_: pvnet.models.multimodal.multimodal.Model
+
+output_quantiles: [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]
+
+#--------------------------------------------
+# NWP encoder
+#--------------------------------------------
+
+nwp_encoders_dict:
+  ukv:
+    _target_: pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet
+    _partial_: True
+    in_channels: 2
+    out_features: 256
+    number_of_conv3d_layers: 6
+    conv3d_channels: 32
+    image_size_pixels: 24
+  ecmwf:
+    _target_: pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet
+    _partial_: True
+    in_channels: 12
+    out_features: 256
+    number_of_conv3d_layers: 4
+    conv3d_channels: 32
+    image_size_pixels: 12
+
+#--------------------------------------------
+# Sat encoder settings
+#--------------------------------------------
+
+sat_encoder:
+  _target_: pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet
+  _partial_: True
+  in_channels: 11
+  out_features: 256
+  number_of_conv3d_layers: 6
+  conv3d_channels: 32
+  image_size_pixels: 24
+
+add_image_embedding_channel: False
+
+#--------------------------------------------
+# PV encoder settings
+#--------------------------------------------
+
+pv_encoder:
+  _target_: pvnet.models.multimodal.site_encoders.encoders.SingleAttentionNetwork
+  _partial_: True
+  num_sites: 349
+  out_features: 40
+  num_heads: 4
+  kdim: 40
+  id_embed_dim: 20
+
+#--------------------------------------------
+# Tabular network settings
+#--------------------------------------------
+
+output_network:
+  _target_: pvnet.models.multimodal.linear_networks.networks.ResFCNet2
+  _partial_: True
+  fc_hidden_features: 128
+  n_res_blocks: 6
+  res_block_layers: 2
+  dropout_frac: 0.0
+
+embedding_dim: 16
+include_sun: True
+include_gsp_yield_history: False
+include_site_yield_history: False
+
+# The mapping between the location IDs and their embedding indices
+location_id_mapping:
+  1: 1
+  5: 2
+  110: 3
+# ...
+
+#--------------------------------------------
+# Times
+#--------------------------------------------
+
+# Foreast and time settings
+forecast_minutes: 480
+history_minutes: 120
+
+min_sat_delay_minutes: 60
+
+# These must also be set even if identical to forecast_minutes and  history_minutes
+sat_history_minutes: 90
+pv_history_minutes: 180
+
+# These must be set for each NWP encoder
+nwp_history_minutes:
+  ukv: 120
+  ecmwf: 120
+nwp_forecast_minutes:
+  ukv: 480
+  ecmwf: 480
+# Optional; defaults to 60, so must be set for data with different time resolution
+nwp_interval_minutes:
+  ukv: 60
+  ecmwf: 60
+
+# ----------------------------------------------
+# Optimizer
+# ----------------------------------------------
+optimizer:
+  _target_: pvnet.optimizers.EmbAdamWReduceLROnPlateau
+  lr: 0.0001
+  weight_decay: 0.01
+  amsgrad: True
+  patience: 5
+  factor: 0.1
+  threshold: 0.002

configs.example/model/nwp_dwsrf_weighting.yaml

@@ -0,0 +1,21 @@
+_target_: pvnet.models.multimodal.nwp_weighting.Model
+
+#--------------------------------------------
+# Network settings
+#--------------------------------------------
+
+# Foreast and time settings
+forecast_minutes: 480
+history_minutes: 120
+
+nwp_history_minutes: 120
+nwp_forecast_minutes: 480
+
+nwp_image_size_pixels: 24
+dwsrf_channel: 1
+
+# ----------------------------------------------
+
+optimizer:
+  _target_: pvnet.optimizers.AdamW
+  lr: 0.0005

configs.example/model/test.yaml

@@ -0,0 +1,4 @@
+_target_: pvnet.models.baseline.single_value.Model
+
+history_minutes: 120
+forecast_minutes: 360

configs.example/model/wind_multimodal.yaml

@@ -0,0 +1,83 @@
+_target_: pvnet.models.multimodal.multimodal.Model
+
+output_quantiles: [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]
+
+#--------------------------------------------
+# NWP encoder
+#--------------------------------------------
+nwp_encoders_dict:
+  ecmwf:
+    _target_: pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet
+    _partial_: True
+    in_channels: 14
+    out_features: 256
+    number_of_conv3d_layers: 6
+    conv3d_channels: 32
+    image_size_pixels: 16
+
+#--------------------------------------------
+# Sensor encoder settings
+#--------------------------------------------
+
+wind_encoder:
+  _target_: pvnet.models.multimodal.site_encoders.encoders.SingleAttentionNetwork
+  _partial_: True
+  num_sites: 19
+  out_features: 40
+  num_heads: 4
+  kdim: 40
+  id_embed_dim: 20
+
+#--------------------------------------------
+# Tabular network settings
+#--------------------------------------------
+
+output_network:
+  _target_: pvnet.models.multimodal.linear_networks.networks.ResFCNet2
+  _partial_: True
+  fc_hidden_features: 128
+  n_res_blocks: 6
+  res_block_layers: 2
+  dropout_frac: 0.0
+
+embedding_dim: 16
+include_sun: False
+include_gsp_yield_history: False
+
+# The mapping between the location IDs and their embedding indices
+location_id_mapping:
+  1: 1
+  5: 2
+  110: 3
+# ...
+
+#--------------------------------------------
+# Times
+#--------------------------------------------
+
+# Foreast and time settings
+forecast_minutes: 480
+history_minutes: 120
+
+min_sat_delay_minutes: 60
+
+# --- set to null if same as history_minutes ---
+sat_history_minutes: 90
+nwp_history_minutes: 60
+nwp_forecast_minutes: 2880
+pv_history_minutes: 180
+pv_interval_minutes: 15
+sat_interval_minutes: 15
+
+target_key: "sensor"
+# ----------------------------------------------
+# Optimizer
+# ----------------------------------------------
+optimizer:
+  _target_: pvnet.optimizers.EmbAdamWReduceLROnPlateau
+  lr: 0.0001
+  weight_decay: 0.01
+  amsgrad: True
+  patience: 5
+  factor: 0.1
+  threshold: 0.002

configs.example/readme.md

@@ -0,0 +1,5 @@
+This directory contains example configuration files for the PVNet project. Many paths will need to unique to each user. You can find these paths by searching for PLACEHOLDER within these logs. Not all of
+the values with a placeholder need to be set. For example in the logger subdirectory there are many different loggers with PLACEHOLDERS. If only one logger is used, then only that placeholder needs to be set.
+
+run experiments by:
+`python run.py experiment=example_simple `

configs.example/trainer/all_params.yaml

@@ -0,0 +1,48 @@
+_target_: pytorch_lightning.Trainer
+
+# default values for all trainer parameters
+checkpoint_callback: True
+default_root_dir: null
+gradient_clip_val: 0.0
+process_position: 0
+num_nodes: 1
+num_processes: 1
+gpus: null
+auto_select_gpus: False
+tpu_cores: null
+log_gpu_memory: null
+overfit_batches: 0.0
+track_grad_norm: -1
+check_val_every_n_epoch: 1
+fast_dev_run: False
+accumulate_grad_batches: 1
+max_epochs: 1
+min_epochs: 1
+max_steps: null
+min_steps: null
+limit_train_batches: 1.0
+limit_val_batches: 1.0
+limit_test_batches: 1.0
+val_check_interval: 1.0
+flush_logs_every_n_steps: 100
+log_every_n_steps: 50
+accelerator: null
+sync_batchnorm: False
+precision: 32
+weights_save_path: null
+num_sanity_val_steps: 2
+truncated_bptt_steps: null
+resume_from_checkpoint: null
+profiler: null
+benchmark: False
+deterministic: False
+reload_dataloaders_every_epoch: False
+auto_lr_find: False
+replace_sampler_ddp: True
+terminate_on_nan: False
+auto_scale_batch_size: False
+prepare_data_per_node: True
+plugins: null
+amp_backend: "native"
+amp_level: "O2"
+move_metrics_to_cpu: False

configs.example/trainer/default.yaml

@@ -0,0 +1,14 @@
+_target_: lightning.pytorch.trainer.trainer.Trainer
+
+# set `1` to train on GPU, `0` to train on CPU only
+accelerator: auto
+devices: auto
+
+min_epochs: null
+max_epochs: null
+reload_dataloaders_every_n_epochs: 0
+num_sanity_val_steps: 8
+fast_dev_run: false
+
+accumulate_grad_batches: 4
+log_every_n_steps: 50

data_config.yaml

@@ -27,10 +27,9 @@ input_data:
       dropout_fraction: 1.0
       dropout_timedeltas_minutes:
       - -180
-      forecast_minutes: 3120.0
-      image_size_pixels_height: 36
-      image_size_pixels_width: 36
-      interval_end_minutes: 2880
+      image_size_pixels_height: 30
+      image_size_pixels_width: 30
+      interval_end_minutes: 2160
       interval_start_minutes: -120
       max_staleness_minutes: null
       normalisation_constants:
@@ -124,11 +123,10 @@ input_data:
     - -20
     - -25
     - -30
-    image_size_pixels_height: 144
-    image_size_pixels_width: 144
-    interval_end_minutes: 0
+    image_size_pixels_height: 100
+    image_size_pixels_width: 100
+    interval_end_minutes: -60
     interval_start_minutes: -90
-    live_delay_minutes: 60
     normalisation_constants:
       HRV:
         mean: 0.09298719
@@ -166,18 +164,16 @@ input_data:
       WV_073:
         mean: 0.62479186
         std: 0.12924142
-    satellite_image_size_pixels_height: 144
-    satellite_image_size_pixels_width: 144
     time_resolution_minutes: 5
     zarr_path: PLACEHOLDER.zarr
   site:
     capacity_mode: variable
     file_path: /home/zak/projects/PVNet/nl_solar/pv_data/netherlands_pv_data_v2.nc
-    interval_end_minutes: 2880
+    interval_end_minutes: 2160
     interval_start_minutes: -2880
     metadata_file_path: /home/zak/projects/PVNet/nl_solar/pv_data/netherlands_metadata.csv
     time_resolution_minutes: 15
   solar_position:
-    interval_end_minutes: 2880
+    interval_end_minutes: 2160
     interval_start_minutes: -2880
     time_resolution_minutes: 15

experiments/india/001_v1/india_pv_wind.md

@@ -0,0 +1,69 @@
+# PVNet for Wind and PV Sites in India
+
+## PVNet for sites
+
+### Data
+
+We use PV generation data for India from April 2019-Nov 2022 for training
+and Dec 2022- Nov 2023 for validation. This is only with ECMWF data, and PV generation history.
+
+The forecast is every 15 minutes for 48 hours for PV generation.
+
+The input NWP data is hourly, and 32x32 pixels (corresponding to around 320kmx320km) around a central
+point in NW-India.
+
+[WandB Link](https://wandb.ai/openclimatefix/pvnet_india2.1/runs/o4xpvzrc)
+
+### Results
+
+Overall MAE is 4.9% on the validation set, and forecasts look overall good.
+
+![batch_idx_1_all_892_2ca7e12db5de2cf2e244](https://github.com/openclimatefix/PVNet/assets/7170359/07e8199a-11b5-4400-9897-37b7738a4f39)
+
+![W B Chart 05_02_2024, 10_07_12_pvnet](https://github.com/openclimatefix/PVNet/assets/7170359/abaefdc1-dedd-4a12-8a26-afaf36d7786b)
+
+## WindNet
+
+
+### April-29-2024 WindNet v1 Production Model
+
+[WandB Link](https://wandb.ai/openclimatefix/india/runs/5llq8iw6)
+
+Improvements: Larger input size (64x64), 7 hour delay for ECMWF NWP inputs, to match productions.
+New, much more efficient encoder for NWP, allowing for more filters and layers, with less parameters.
+The 64x64 input size corresponds to 6.4 degrees x 6.4 degrees, which is around 700km x 700km. This allows for the
+model to see the wind over the wind generation sites, which seems to be the biggest reason for the improvement in the model.
+
+
+
+MAE is 7.6% with real improvements on the production side of things.
+
+
+There were other experiments with slightly different numbers of filters, model parameters and the like, but generally no
+improvements were seen.
+
+
+## WindNet v1 Results
+
+### Data
+
+We use Wind generation data for India from April 2019-Nov 2022 for training
+and Dec 2022- Nov 2023 for validation. This is only with ECMWF data, and Wind generation history.
+
+The forecast is every 15 minutes for 48 hours for Wind generation.
+
+The input NWP data is hourly, and 32x32 pixels (corresponding to around 320kmx320km) around a central
+point in NW-India. Note: The majority of the wind generation is likely not covered in the 320kmx320km area.
+
+
+[WandB Link](https://wandb.ai/openclimatefix/pvnet_india2.1/runs/otdx7axx)
+
+### Results
+
+![W B Chart 05_02_2024, 10_05_19](https://github.com/openclimatefix/PVNet/assets/7170359/6a8cd9c5-bdfe-41ab-996d-37fd1be2a07c)
+
+![W B Chart 05_02_2024, 10_06_51_windnet](https://github.com/openclimatefix/PVNet/assets/7170359/77554ef0-4411-4432-af95-8530aef4a701)
+
+![batch_idx_1_all_1730_379a9f881a7f01153f98](https://github.com/openclimatefix/PVNet/assets/7170359/243d9f3e-4cb9-405e-80c5-40c6c218c17f)
+
+MAE is around 10% overall, although it doesn't seem to do very well on the ramps up and down.

experiments/india/002_wind_meteomatics/india_windnet_v2.md

@@ -0,0 +1,46 @@
+### WindNet v2 Meteomatics + ECMWF Model
+
+[WandB Linl](https://wandb.ai/openclimatefix/india/runs/v3mja33d)
+
+This newest experiment uses Meteomatics data in addition to ECMWF data. The Meteomatics data is at specific locations corresponding
+to the gneeration sites we know about. It is smartly downscaled ECMWF data, down to 15 minutes and at a few height levels we are
+interested in, primarily 10m, 100m, and 200m. The Meteomatics data is a semi-reanalysis, with each block of 6 hours being from one forecast run.
+For example, in one day, hours 00-06 are from the same, 00 forecast run, and hours 06-12 are from the 06 forecast run. This is important to note
+as it is both not a real reanalysis, but we also can't have it exactly match the live data, as any forecast steps beyond 6 hours are thrown away.
+This does mean that these results should be taken as a best case or better than best case scenario, as every 6 hour, observations from the future
+are incorporated into the Meteomatics input data from the next NWP mode run.
+
+For the purposes of WindNet, Meteomatics data is treated as Sensor data that goes into the future.
+The model encodes the sensor information the same way as for the historical PV, Wind, and GSP generation, and has
+a simple, single attention head to encode the information. This is then concatenated along with the rest of the data, like in
+previous experiments.
+
+This model also has an even larger input size of ECMWF data, 81x81 pixels, corresponding to around 810kmx810km.
+![Screenshot_20240430_082855](https://github.com/openclimatefix/PVNet/assets/7170359/6981a088-8664-474b-bfea-c94c777fc119)
+
+MAE is 7.0% on the validation set, showing a slight improvement over the previous model.
+
+Comperison  with the production model:
+
+| Timestep | Prod MAE % | No Meteomatics MAE % | Meteomatics MAE % |
+| --- | --- | --- | --- |
+| 0-0 minutes | 7.586 | 5.920 | 2.475 |
+| 15-15 minutes | 8.021 | 5.809 | 2.968 |
+| 30-45 minutes | 7.233 | 5.742 | 3.472 |
+| 45-60 minutes | 7.187 | 5.698 | 3.804 |
+| 60-120 minutes | 7.231 | 5.816 | 4.650 |
+| 120-240 minutes | 7.287 | 6.080 | 6.028 |
+| 240-360 minutes | 7.319 | 6.375 | 6.738 |
+| 360-480 minutes | 7.285 | 6.638 | 6.964 |
+| 480-720 minutes | 7.143 | 6.747 | 6.906 |
+| 720-1440 minutes | 7.380 | 7.207 | 6.962 |
+| 1440-2880 minutes | 7.904 | 7.507 | 7.507 |
+
+![mae_per_timestep](https://github.com/openclimatefix/PVNet/assets/7170359/e3c942e8-65c6-4b95-8c51-f25d43e7a082)
+
+
+
+
+Example plot
+
+![Screenshot_20240430_082937](https://github.com/openclimatefix/PVNet/assets/7170359/88db342e-bf82-414e-8255-5ad4af659fb8)

experiments/india/003_wind_plevels/plevel.md

@@ -0,0 +1,54 @@
+# Running WindNet for RUVNL for diferent Plevels
+
+https://wandb.ai/openclimatefix/india/runs/5llq8iw6 is the current production one
+This has 7 plevels and a small patch size.
+
+## Experiments
+
+1. Only used plevel 50 (orange)
+https://wandb.ai/openclimatefix/india/runs/ziudzweq/
+
+2. Use plevels of [2, 10, 25, 50, 75, 90, 98]. This is what is already used. (green)
+https://wandb.ai/openclimatefix/india/runs/xdlew7ib
+
+3. Use plevels of [1, 02, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80 (brown)
+, 90, 98, 99]
+https://wandb.ai/openclimatefix/india/runs/pcr2zsrc
+
+
+## Training
+
+Each epoch took about ~4 hours, so the training runs took several days.
+
+TODO add number of samples
+
+## Results
+
+MAE results show that using the plevel of 50 only, gives better results
+![](Mae.png "Mae")
+
+The p50 results are about the same
+![](p50.png "p50")
+
+We can see that for p10 the results are not right, as they should converge to 0.1
+![](p10.png "p10")
+
+Interestingly the more plevels you have the better the results are for before 4 hours
+but the less plevels you have the better the results for >= 8 hours.
+
+| Timestep | P50 only MAE % | 7 plevels MAE % | 15 plevel MAE %  | 7 plevels small patch MAE % |
+| --- | --- | --- |  --- | --- |
+| 0-0 minutes | 5.416 | 5.920 | 3.933 | 7.586 |
+| 15-15 minutes | 5.458 | 5.809 | 4.003 | 8.021 |
+| 30-45 minutes | 5.525 | 5.742 | 4.442 | 7.233 |
+| 45-60 minutes | 5.595 | 5.698 | 4.772 | 7.187 |
+| 60-120 minutes | 5.890 | 5.816 | 5.307 | 7.231 |
+| 120-240 minutes | 6.423 | 6.080 | 6.275 | 7.287 |
+| 240-360 minutes | 6.608 | 6.375 | 6.707 | 7.319 |
+| 360-480 minutes | 6.728 | 6.638 | 6.904 | 7.285 |
+| 480-720 minutes | 6.634 | 6.747 | 6.872 | 7.143 |
+| 720-1440 minutes | 6.940 | 7.207 | 7.176 | 7.380 |
+| 1440-2880 minutes | 7.446 | 7.507 | 7.735 | 7.904 |
+
+
+![](MAEvstimesteps.png "MAEvstimesteps")

experiments/india/004_n_training_samples/log-plot.py

@@ -0,0 +1,14 @@
+""" Small script to make MAE vs number of batches plot"""
+
+import pandas as df
+import plotly.graph_objects as go
+
+data = [[100, 7.779], [300, 7.441], [1000, 7.181], [3000, 7.180], [6711, 7.151]]
+df = df.DataFrame(data, columns=["n_samples", "MAE [%]"])
+
+fig = go.Figure()
+fig.add_trace(go.Scatter(x=df["n_samples"], y=df["MAE [%]"], mode="lines+markers"))
+fig.update_layout(title="MAE % for N samples", xaxis_title="N Samples", yaxis_title="MAE %")
+# change to log log
+fig.update_xaxes(type="log")
+fig.show(renderer="browser")

experiments/india/004_n_training_samples/readme.md

@@ -0,0 +1,48 @@
+# N samples experiments
+
+Kicked off an experiment that uses N samples
+This is done by adding `limit_train_batches` to the `trainer/default.yaml`.
+
+I checked that when limiting the batches, the same batches are shown to model for each epoch.
+
+## Experiments
+
+Original is 6711 batches
+
+- 100: 3p6scx2r
+- 300: am46tno1
+- 1000: u04xlb6p
+- 3000: p11lhreo
+
+## Results
+
+Overall
+
+| Experiment | MAE % |
+|------------|-------|
+| 100        | 7.779 |
+| 300        | 7.441 |
+| 1000       | 7.181 |
+| 3000       | 7.180 |
+| 6711       | 7.151 |
+
+Results by timestamps
+
+
+| Timestep | 100 MAE % | 300 MAE % | 1000 MAE % | 3000 MAE % | 6711 MAE % |
+| --- | --- | --- | --- | --- | --- |
+| 0-0 minutes | 7.985 | 7.453 | 7.155 | 5.553 | 5.920 |
+| 15-15 minutes | 7.953 | 7.055 | 6.923 | 5.453 | 5.809 |
+| 30-45 minutes | 8.043 | 7.172 | 6.907 | 5.764 | 5.742 |
+| 45-60 minutes | 7.850 | 7.070 | 6.790 | 5.815 | 5.698 |
+| 60-120 minutes | 7.698 | 6.809 | 6.597 | 5.890 | 5.816 |
+| 120-240 minutes | 7.355 | 6.629 | 6.495 | 6.221 | 6.080 |
+| 240-360 minutes | 7.230 | 6.729 | 6.559 | 6.541 | 6.375 |
+| 360-480 minutes | 7.415 | 6.997 | 6.770 | 6.855 | 6.638 |
+| 480-720 minutes | 7.258 | 7.037 | 6.668 | 6.876 | 6.747 |
+| 720-1440 minutes | 7.659 | 7.362 | 7.038 | 7.142 | 7.207 |
+| 1440-2880 minutes | 8.027 | 7.745 | 7.518 | 7.535 | 7.507 |
+
+![](mae_step.png "mae_steps")
+
+![](mae_samples.png "mae_samples")