Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000200.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000400.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000600.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000800.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001000.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001200.json +127 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001400.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001600.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001800.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002000.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002200.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002400.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002600.json +144 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000200.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000400.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000600.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000800.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001000.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001200.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001400.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001600.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001800.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002000.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002200.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002400.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002600.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank0.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank1.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank2.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank3.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank4.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank5.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank6.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank0.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank1.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank2.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank3.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank4.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank5.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank6.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank0.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank1.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank2.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank3.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank4.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank5.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank6.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank0.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank1.pt +3 -0
- fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank2.pt +3 -0
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000200.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 200,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 4012,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.016370560973882675,
|
| 124 |
+
"smooth_train_loss": 0.5858559630265132,
|
| 125 |
+
"total_training_time": 671.0913317203522
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000400.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 400,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 7490,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.013834112323820591,
|
| 124 |
+
"smooth_train_loss": 0.5782125935767392,
|
| 125 |
+
"total_training_time": 1311.291844367981
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000600.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 600,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 10974,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.010902605019509792,
|
| 124 |
+
"smooth_train_loss": 0.6522089063121922,
|
| 125 |
+
"total_training_time": 2001.2056035995483
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_000800.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 800,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 14454,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.010902605019509792,
|
| 124 |
+
"smooth_train_loss": 0.5903836969554602,
|
| 125 |
+
"total_training_time": 2712.8702449798584
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001000.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1000,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 17963,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.010902605019509792,
|
| 124 |
+
"smooth_train_loss": 0.6888835440006619,
|
| 125 |
+
"total_training_time": 3383.462060689926
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001200.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1200,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast",
|
| 57 |
+
"device_type": "",
|
| 58 |
+
"num_iterations": 2000,
|
| 59 |
+
"device_batch_size": 2,
|
| 60 |
+
"grad_accum_steps": 8,
|
| 61 |
+
"lr": 0.0001,
|
| 62 |
+
"weight_decay": 0.05,
|
| 63 |
+
"adam_beta1": 0.9,
|
| 64 |
+
"adam_beta2": 0.95,
|
| 65 |
+
"warmup_ratio": 0.01,
|
| 66 |
+
"warmdown_ratio": 0.4,
|
| 67 |
+
"final_lr_frac": 0.0,
|
| 68 |
+
"grad_clip": 1.0,
|
| 69 |
+
"resume_from_step": -1,
|
| 70 |
+
"eval_every": 200,
|
| 71 |
+
"eval_steps": 25,
|
| 72 |
+
"save_every": 200,
|
| 73 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 74 |
+
},
|
| 75 |
+
"dataset_meta": {
|
| 76 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 77 |
+
"s2_key": "sentinel2_l2a",
|
| 78 |
+
"s1_key": "sentinel1",
|
| 79 |
+
"landsat_key": "landsat",
|
| 80 |
+
"num_timesteps": 12,
|
| 81 |
+
"timestep_strategy": "uniform",
|
| 82 |
+
"patch_size": 16,
|
| 83 |
+
"grid_size": 8,
|
| 84 |
+
"mask_ratio": 0.75,
|
| 85 |
+
"value_scales": {
|
| 86 |
+
"sentinel2_l2a": 0.0001,
|
| 87 |
+
"sentinel1": 1.0,
|
| 88 |
+
"landsat": 0.0001
|
| 89 |
+
},
|
| 90 |
+
"norm": {
|
| 91 |
+
"strategy": "computed",
|
| 92 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 93 |
+
"std_multiplier": 2.0
|
| 94 |
+
},
|
| 95 |
+
"modalities": [
|
| 96 |
+
"sentinel2_l2a",
|
| 97 |
+
"sentinel1",
|
| 98 |
+
"landsat"
|
| 99 |
+
],
|
| 100 |
+
"modality_channels": {
|
| 101 |
+
"sentinel2_l2a": 12,
|
| 102 |
+
"sentinel1": 2,
|
| 103 |
+
"landsat": 11
|
| 104 |
+
},
|
| 105 |
+
"contrastive": {
|
| 106 |
+
"weight": 0.1,
|
| 107 |
+
"temp": 0.1,
|
| 108 |
+
"proj_dim": 256,
|
| 109 |
+
"queue_size": 4096,
|
| 110 |
+
"pooling": "mean_unmasked",
|
| 111 |
+
"spatial_aug": "flip_rotate",
|
| 112 |
+
"drop_s1_p": 0.3,
|
| 113 |
+
"drop_landsat_p": 0.3
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
"grad_accum_steps": 8,
|
| 117 |
+
"dataloader_state_dict": {
|
| 118 |
+
"epoch": 1,
|
| 119 |
+
"seen_in_epoch": 21405,
|
| 120 |
+
"rank": 0
|
| 121 |
+
},
|
| 122 |
+
"loop_state": {
|
| 123 |
+
"min_val_loss": 0.010902605019509792,
|
| 124 |
+
"smooth_train_loss": 0.6212157924077802,
|
| 125 |
+
"total_training_time": 4036.29727101326
|
| 126 |
+
}
|
| 127 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001400.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1400,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 1,
|
| 136 |
+
"seen_in_epoch": 25428,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.006192712113261223,
|
| 141 |
+
"smooth_train_loss": 0.6052941592953123,
|
| 142 |
+
"total_training_time": 1029.8920831680298
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001600.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1600,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 1,
|
| 136 |
+
"seen_in_epoch": 28896,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.005149390082806349,
|
| 141 |
+
"smooth_train_loss": 0.646685980323255,
|
| 142 |
+
"total_training_time": 1955.4804801940918
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_001800.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1800,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 2,
|
| 136 |
+
"seen_in_epoch": 917,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.005149390082806349,
|
| 141 |
+
"smooth_train_loss": 0.6704386385443244,
|
| 142 |
+
"total_training_time": 2887.5464861392975
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002000.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2000,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 2,
|
| 136 |
+
"seen_in_epoch": 4390,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.0049408539198338985,
|
| 141 |
+
"smooth_train_loss": 0.6197526026832074,
|
| 142 |
+
"total_training_time": 3779.869448184967
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002200.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2200,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 2,
|
| 136 |
+
"seen_in_epoch": 7863,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.0049408539198338985,
|
| 141 |
+
"smooth_train_loss": 0.7462502316266328,
|
| 142 |
+
"total_training_time": 4642.677878856659
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002400.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2400,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 2,
|
| 136 |
+
"seen_in_epoch": 11335,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.004638022743165493,
|
| 141 |
+
"smooth_train_loss": 0.7643059450641918,
|
| 142 |
+
"total_training_time": 5606.75569486618
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/meta_002600.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2600,
|
| 3 |
+
"model_type": "fm_vit",
|
| 4 |
+
"model_config": {
|
| 5 |
+
"embed_dim": 768,
|
| 6 |
+
"depth": 12,
|
| 7 |
+
"num_heads": 12,
|
| 8 |
+
"mlp_ratio": 4.0,
|
| 9 |
+
"dropout": 0.0,
|
| 10 |
+
"attn_dropout": 0.0,
|
| 11 |
+
"patch_size": 16,
|
| 12 |
+
"modalities": [
|
| 13 |
+
"sentinel2_l2a",
|
| 14 |
+
"sentinel1",
|
| 15 |
+
"landsat"
|
| 16 |
+
],
|
| 17 |
+
"modality_channels": {
|
| 18 |
+
"sentinel2_l2a": 12,
|
| 19 |
+
"sentinel1": 2,
|
| 20 |
+
"landsat": 11
|
| 21 |
+
}
|
| 22 |
+
},
|
| 23 |
+
"user_config": {
|
| 24 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 25 |
+
"s2_key": "sentinel2_l2a",
|
| 26 |
+
"s1_key": "sentinel1",
|
| 27 |
+
"landsat_key": "landsat",
|
| 28 |
+
"num_timesteps": 12,
|
| 29 |
+
"timestep_strategy": "uniform",
|
| 30 |
+
"patch_size": 16,
|
| 31 |
+
"mask_ratio": 0.75,
|
| 32 |
+
"s2_value_scale": 0.0001,
|
| 33 |
+
"s1_value_scale": 1.0,
|
| 34 |
+
"landsat_value_scale": 0.0001,
|
| 35 |
+
"norm": "computed",
|
| 36 |
+
"norm_stats_file": "",
|
| 37 |
+
"norm_std_multiplier": 2.0,
|
| 38 |
+
"num_workers": 120,
|
| 39 |
+
"prefetch": 512,
|
| 40 |
+
"shuffle_buffer": 10000,
|
| 41 |
+
"depth": 12,
|
| 42 |
+
"embed_dim": 768,
|
| 43 |
+
"num_heads": 12,
|
| 44 |
+
"mlp_ratio": 4.0,
|
| 45 |
+
"dropout": 0.0,
|
| 46 |
+
"attn_dropout": 0.0,
|
| 47 |
+
"contrastive_weight": 0.1,
|
| 48 |
+
"contrastive_temp": 0.1,
|
| 49 |
+
"contrastive_proj_dim": 256,
|
| 50 |
+
"contrastive_lr": 0.001,
|
| 51 |
+
"contrastive_queue_size": 4096,
|
| 52 |
+
"contrastive_pooling": "mean_unmasked",
|
| 53 |
+
"spatial_aug": "flip_rotate",
|
| 54 |
+
"drop_s1_p": 0.3,
|
| 55 |
+
"drop_landsat_p": 0.3,
|
| 56 |
+
"distill_weight": 1.0,
|
| 57 |
+
"distill_every": 1,
|
| 58 |
+
"distill_teacher_patch_size": 4,
|
| 59 |
+
"distill_teacher_expected_hw": 64,
|
| 60 |
+
"distill_teacher_timesteps": 1,
|
| 61 |
+
"distill_olmoearth_src_dir": "/workspace/Spatial/olmoearth_pretrain",
|
| 62 |
+
"distill_olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 63 |
+
"distill_olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth",
|
| 64 |
+
"run": "fmvit_d12_e768_ps16_normcomputed_contrast_distill_from1200_fixed",
|
| 65 |
+
"device_type": "",
|
| 66 |
+
"num_iterations": 3000,
|
| 67 |
+
"device_batch_size": 2,
|
| 68 |
+
"grad_accum_steps": 8,
|
| 69 |
+
"lr": 0.0001,
|
| 70 |
+
"weight_decay": 0.05,
|
| 71 |
+
"adam_beta1": 0.9,
|
| 72 |
+
"adam_beta2": 0.95,
|
| 73 |
+
"warmup_ratio": 0.01,
|
| 74 |
+
"warmdown_ratio": 0.4,
|
| 75 |
+
"final_lr_frac": 0.0,
|
| 76 |
+
"grad_clip": 1.0,
|
| 77 |
+
"resume_from_step": 1200,
|
| 78 |
+
"eval_every": 200,
|
| 79 |
+
"eval_steps": 25,
|
| 80 |
+
"save_every": 200,
|
| 81 |
+
"model_tag": "fmvit_d12_e768_ps16_normcomputed_contrast"
|
| 82 |
+
},
|
| 83 |
+
"dataset_meta": {
|
| 84 |
+
"h5_dir": "/workspace/olmoearth/h5_shards",
|
| 85 |
+
"s2_key": "sentinel2_l2a",
|
| 86 |
+
"s1_key": "sentinel1",
|
| 87 |
+
"landsat_key": "landsat",
|
| 88 |
+
"num_timesteps": 12,
|
| 89 |
+
"timestep_strategy": "uniform",
|
| 90 |
+
"patch_size": 16,
|
| 91 |
+
"grid_size": 8,
|
| 92 |
+
"mask_ratio": 0.75,
|
| 93 |
+
"value_scales": {
|
| 94 |
+
"sentinel2_l2a": 0.0001,
|
| 95 |
+
"sentinel1": 1.0,
|
| 96 |
+
"landsat": 0.0001
|
| 97 |
+
},
|
| 98 |
+
"norm": {
|
| 99 |
+
"strategy": "computed",
|
| 100 |
+
"stats_file": "/workspace/Spatial/olmoearth_pretrain/olmoearth_pretrain/data/norm_configs/computed.json",
|
| 101 |
+
"std_multiplier": 2.0
|
| 102 |
+
},
|
| 103 |
+
"modalities": [
|
| 104 |
+
"sentinel2_l2a",
|
| 105 |
+
"sentinel1",
|
| 106 |
+
"landsat"
|
| 107 |
+
],
|
| 108 |
+
"modality_channels": {
|
| 109 |
+
"sentinel2_l2a": 12,
|
| 110 |
+
"sentinel1": 2,
|
| 111 |
+
"landsat": 11
|
| 112 |
+
},
|
| 113 |
+
"contrastive": {
|
| 114 |
+
"weight": 0.1,
|
| 115 |
+
"temp": 0.1,
|
| 116 |
+
"proj_dim": 256,
|
| 117 |
+
"queue_size": 4096,
|
| 118 |
+
"pooling": "mean_unmasked",
|
| 119 |
+
"spatial_aug": "flip_rotate",
|
| 120 |
+
"drop_s1_p": 0.3,
|
| 121 |
+
"drop_landsat_p": 0.3
|
| 122 |
+
},
|
| 123 |
+
"distill": {
|
| 124 |
+
"weight": 1.0,
|
| 125 |
+
"every": 1,
|
| 126 |
+
"teacher_patch_size": 4,
|
| 127 |
+
"teacher_expected_hw": 64,
|
| 128 |
+
"teacher_timesteps": 1,
|
| 129 |
+
"olmoearth_config": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/config.json",
|
| 130 |
+
"olmoearth_weights": "/workspace/Spatial/nanochat_artifacts/teachers/olmoearth_v1_base/weights.pth"
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"grad_accum_steps": 8,
|
| 134 |
+
"dataloader_state_dict": {
|
| 135 |
+
"epoch": 2,
|
| 136 |
+
"seen_in_epoch": 14836,
|
| 137 |
+
"rank": 0
|
| 138 |
+
},
|
| 139 |
+
"loop_state": {
|
| 140 |
+
"min_val_loss": 0.004481656476855278,
|
| 141 |
+
"smooth_train_loss": 0.7224110388834869,
|
| 142 |
+
"total_training_time": 6496.724760055542
|
| 143 |
+
}
|
| 144 |
+
}
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000200.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:56c5811d319d15b3fdc64c67f2c88e9bcbff4f21738bf6be002eed11ee0515af
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000400.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5c1e22159d25baf272d45a99a2f008b477c3113e90ae93239e5ec88d29b29bf
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000600.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4df6f9880262708099f19c44a57a1b6bd2c59678dcd8ba52161bdfcf8dba17f5
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_000800.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:79e9caec74d6b4498589b3c0f70d934eb7cc74d5690676e71e24153453516ad4
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:32876e28b75c481fee03c9df17914fd4c1433f6dded65d3ac3675dfe44b45477
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001200.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ee9204f10851764e37b4f95f473584f4bdb1e780fb35f8f3d7c62cea1500fbdb
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001400.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa834c3118880dca8e7795fe2c395a998b629b014a480264efe63a9053ea8360
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001600.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:70f11504e29f22a69f72eace4eda0134ec64fbe05ee4c323efa52bdbea2f40cd
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_001800.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:00ef2c386940ae7d194d85a3357b8460bf3adfc095ee6cc5c08291e74a1521dc
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:51311e1e165af3a35daa4abbd4637ceaa5f9954f8bf75984a41d6f6ab6f86174
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002200.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:461a2302182534751ea374a92b77e8d2ae42645e682302eea2ead960cf0c5ff1
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002400.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bfa4572b2190c86e6887b5c62f56d3889e97c259938a463b9b96ebc6d22ede79
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/model_002600.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c7bddeba2ffbfacae1b5cdf116b78cc266948da561effb8e213e57204a9e0ee8
|
| 3 |
+
size 606639149
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dbf41cecc21436ecd74c2b782279412d6200e93f6374fe115f553d21d5b99c01
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0fa5ae20454acaf2a80f078cae9241ef86d7715cc68d2efb20b48a87f50e4ab8
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0306c06096d6dabd45eb7ec000f76f5e4b45e481fdf9088237c4f0c2a99ff5da
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:246bccba91f30dc9e71a1797f1aaa9f4b1e31a362852c53c1ee49776783b1cb6
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank4.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:384396ad982a665c05bde9cae27ef34494c31e2e275635775e4cfa9c3c0d1259
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank5.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f2495201e28ff9c44887a702e1e50b3de33ef7d1fce8cb1f42ec317e6baafd92
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000200_rank6.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9ad73e99462d90e0edcf9860d7df97db6ece8436722c55e8a2cfe8d96d34ac20
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a28088649cb83430f95a342a859f9c3106f156879cd84a4f6a148a4215e2a55
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:35c3ab6f24c10244b5d9413cf72eb1dc33aba84704e0bfa7491dce95346d4169
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0fd03658c7a95d0711753e863e559bac746ac4fd8a1c9fca33c277d4a4c25800
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e3b80d94b7667a9aa9de425c7e1db0e829e13fe06c264b9ba3f73eb04aaa5325
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank4.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48302e3094079a8d2966abd699810dfece29fe385d466d2e6be5551629642796
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank5.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc0dde94ba44ae0d3b73c930c4529feb601b2cc053ef523ee553ee39201f8289
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000400_rank6.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f00ea4f96d7e415ac6630efae27bd8fcd35a62c33be6fc7ed6aad1762c389d99
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0096657a9d16f9d7456656af4a3a528111a18889656b82354f8e37d6789a8923
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8f72145f32831f53673ad2972943436cdf10632a74ecb17ddece373335b43166
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a01c547e818ae28bf7b91068f073c84483d0709c4bf6619b39492c5c86a09ee9
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e2a70d662dd2f197e267fcb7ea0ba07db56687aa6cd202641eb80025bea8a39b
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank4.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f01695223c4f4d9afc1c2a7a655811cc978c619234f047d5e350765e0e771e06
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank5.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4c92a9ddda38d164b55c7d703c768e37b0dc6ca9090c99959aa91946e8bbba23
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000600_rank6.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c093d0f3149c6727acc29053a21e67c50296a8c989d2446355e93b672b3d87be
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c95c98030ce612d54364c94f20425c3308d45ca90ce40d485dabfbcca9041d0f
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a77cf216bee93ee96308939620f2e2aa0cb2424b46d8d8708cd08b5c2dba33d5
|
| 3 |
+
size 1215416583
|
fm_checkpoints/fmvit_d12_e768_ps16_normcomputed_contrast/optim_000800_rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:66c7db26f88468a2bd4138b70cdd1dbd34349ddca5494b0f62a399cb982ddc31
|
| 3 |
+
size 1215416583
|