| CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun --nproc_per_node=4 --master_port=8081 \ | |
| train_aurora_15.py \ | |
| --base_dir /hub_data3/minseo/ArchesWeather-SR/data/era5_240/full \ | |
| --train_years 2016-2020 \ | |
| --val_years 2021 \ | |
| --steps 1 \ | |
| --total_steps 10000 \ | |
| --warmup_steps 0 \ | |
| --lr 1e-4 --weight_decay 0 \ | |
| --precision bf16 \ | |
| --batch_size 1 \ | |
| --act_ckpt \ | |
| --disable_droppath \ | |
| --out_dir /hub_data3/minseo/aurora/lora_ckpt_wloss_hres \ | |
| --tb_dir /hub_data3/minseo/aurora/tb_lora_wloss_hres \ | |
| --clim_path /hub_data3/minseo/ArchesWeather-SR/data/era5_240/era5_240_clim.nc \ | |
| --deg_old 1.5 --deg_new 1.5 --win_orig 2,4,8 --win_min_hw 1 --patch_new 1 --patch_old 1 \ | |
| --finetune_mode lora --scheduler const --min_lr_scale 0.0 \ | |
| --lora_mode single --trainable lora_only \ | |
| --surf_stats_json /hub_data3/minseo/aurora/surf_stats_1p5deg.json \ | |
| --surf_stats_days 1000 \ | |
| --ckpt_source local \ | |
| --ckpt_path /hub_data3/minseo/aurora/lora_ckpt_wloss_hres/full_small_final_using.pt \ | |
| # --prune_middle_stages | |
| # --lora_mode single --trainable lora_only \ | |
| # --train_heads --train_token_embed --train_pos_time_embeds \ | |
| # --train_ln_affine_encoder --train_ln_modulation --train_time_mlp |