Commit ·
c8c4766
1
Parent(s): 904ca1b
chore: add trained weights using Git LFS
Browse files- README.md +17 -4
- data/build_dataset_config.yaml +13 -13
- download/hf_download_config.yaml +2 -2
- download/sdo_download_config.yaml +2 -2
- download/sxr_download_config.yaml +1 -1
- training/train_config.yaml +4 -4
README.md
CHANGED
|
@@ -65,7 +65,7 @@ FOXES
|
|
| 65 |
│ ├── convert_aia.py # Raw AIA FITS -> paired 512x512 .npy stacks (itipy)
|
| 66 |
│ ├── combine_sxr.py # Combine raw multi-satellite GOES files into per-satellite CSVs
|
| 67 |
│ ├── align_aia_sxr.py # Match AIA timestamps to GOES CSVs -> per-timestamp SXR .npy
|
| 68 |
-
│ ├── split_train_val_test.py # Split processed AIA/SXR into train/val/test (training only)
|
| 69 |
│ └── sxr_normalization.py # Compute log-space mean/std over SXR .npy files for training
|
| 70 |
├── forecasting
|
| 71 |
│ ├── dataset.py # AIAGOESDataset / AIAGOESDataModule: loads paired AIA + SXR .npy files
|
|
@@ -73,7 +73,8 @@ FOXES
|
|
| 73 |
│ ├── inference.py # Run a checkpoint over a folder of data; writes predictions.csv
|
| 74 |
│ ├── inference_config.yaml # Config for inference.py
|
| 75 |
│ ├── evaluation.py # Compute metrics and generate evaluation plots
|
| 76 |
-
│
|
|
|
|
| 77 |
├── training
|
| 78 |
│ ├── train.py # Train ViTLocal with PyTorch Lightning + Weights & Biases logging
|
| 79 |
│ ├── train_config.yaml # Config for train.py
|
|
@@ -181,14 +182,26 @@ If you don't have ground-truth SXR data (e.g. scoring new/live data), set
|
|
| 181 |
|
| 182 |
### 2) Run inference
|
| 183 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
Edit `forecasting/inference_config.yaml`:
|
| 185 |
|
| 186 |
```yaml
|
| 187 |
data:
|
| 188 |
aia_dir: "/path/to/your/aia_data"
|
| 189 |
sxr_dir: "/path/to/your/sxr_data" # omit/ignore if prediction_only
|
| 190 |
-
sxr_norm_path: "/
|
| 191 |
-
checkpoint_path: "/
|
| 192 |
|
| 193 |
output_path: "/path/to/predictions.csv"
|
| 194 |
```
|
|
|
|
| 65 |
│ ├── convert_aia.py # Raw AIA FITS -> paired 512x512 .npy stacks (itipy)
|
| 66 |
│ ├── combine_sxr.py # Combine raw multi-satellite GOES files into per-satellite CSVs
|
| 67 |
│ ├── align_aia_sxr.py # Match AIA timestamps to GOES CSVs -> per-timestamp SXR .npy
|
| 68 |
+
│ ├── split_train_val_test.py # Split processed AIA/SXR into train/val/test (use for training a new model only)
|
| 69 |
│ └── sxr_normalization.py # Compute log-space mean/std over SXR .npy files for training
|
| 70 |
├── forecasting
|
| 71 |
│ ├── dataset.py # AIAGOESDataset / AIAGOESDataModule: loads paired AIA + SXR .npy files
|
|
|
|
| 73 |
│ ├── inference.py # Run a checkpoint over a folder of data; writes predictions.csv
|
| 74 |
│ ├── inference_config.yaml # Config for inference.py
|
| 75 |
│ ├── evaluation.py # Compute metrics and generate evaluation plots
|
| 76 |
+
│ ├── evaluation_config.yaml # Config for evaluation.py
|
| 77 |
+
│ └── trained_weights_and_normalization/ # 3 released checkpoints + normalized_sxr.npy (see below)
|
| 78 |
├── training
|
| 79 |
│ ├── train.py # Train ViTLocal with PyTorch Lightning + Weights & Biases logging
|
| 80 |
│ ├── train_config.yaml # Config for train.py
|
|
|
|
| 182 |
|
| 183 |
### 2) Run inference
|
| 184 |
|
| 185 |
+
Three released checkpoints are included under
|
| 186 |
+
`forecasting/trained_weights_and_normalization/`, differing only in the
|
| 187 |
+
self-attention mask they were trained with (see `mask_mode` in
|
| 188 |
+
`training/train_config.yaml`) — each checkpoint carries its own mask, so
|
| 189 |
+
nothing else needs to change to switch between them, just `checkpoint_path`:
|
| 190 |
+
|
| 191 |
+
| Checkpoint | `mask_mode` | Description |
|
| 192 |
+
| --- | --- | --- |
|
| 193 |
+
| `inverted-attention-mask.ckpt` | `inverted` | **Original released FOXES model.** Each patch attends to *distant* patches (the flipped local-attention mask it was actually trained with). |
|
| 194 |
+
| `localized-attention-mask.ckpt` | `local` | True local attention — the opposite of the original model: each patch attends only to its own neighborhood. |
|
| 195 |
+
| `no-attention-mask.ckpt` | `none` | Global ViT — standard full/global attention, no masking at all. |
|
| 196 |
+
|
| 197 |
Edit `forecasting/inference_config.yaml`:
|
| 198 |
|
| 199 |
```yaml
|
| 200 |
data:
|
| 201 |
aia_dir: "/path/to/your/aia_data"
|
| 202 |
sxr_dir: "/path/to/your/sxr_data" # omit/ignore if prediction_only
|
| 203 |
+
sxr_norm_path: "forecasting/trained_weights_and_normalization/normalized_sxr.npy"
|
| 204 |
+
checkpoint_path: "forecasting/trained_weights_and_normalization/inverted-attention-mask.ckpt"
|
| 205 |
|
| 206 |
output_path: "/path/to/predictions.csv"
|
| 207 |
```
|
data/build_dataset_config.yaml
CHANGED
|
@@ -10,17 +10,17 @@
|
|
| 10 |
wavelengths: [94, 131, 171, 193, 211, 304, 335]
|
| 11 |
|
| 12 |
aia:
|
| 13 |
-
raw_dir: "/
|
| 14 |
-
bad_files_dir: "/
|
| 15 |
-
processed_dir: "/
|
| 16 |
|
| 17 |
sxr:
|
| 18 |
-
raw_dir: "/
|
| 19 |
-
combined_dir: "/
|
| 20 |
|
| 21 |
output:
|
| 22 |
-
sxr_dir: "/
|
| 23 |
-
aia_missing_dir: "/
|
| 24 |
|
| 25 |
processing:
|
| 26 |
max_processes: null # cap worker processes; null = use all CPUs
|
|
@@ -34,7 +34,7 @@ steps:
|
|
| 34 |
convert_aia: true # raw AIA FITS -> paired .npy stacks
|
| 35 |
combine_sxr: true # combine raw multi-satellite GOES files into per-satellite CSVs
|
| 36 |
align: true # match AIA timestamps to GOES CSVs -> per-timestamp SXR .npy
|
| 37 |
-
split:
|
| 38 |
|
| 39 |
# Only needed if you're training (training/train_config.yaml expects
|
| 40 |
# aia_dir/sxr_dir to each have train/val/test subfolders) — not needed for
|
|
@@ -42,14 +42,14 @@ steps:
|
|
| 42 |
# aia.processed_dir and output.sxr_dir in place, using the same date ranges
|
| 43 |
# so a given timestamp lands in the same split for both.
|
| 44 |
split:
|
| 45 |
-
train_range:
|
| 46 |
-
val_range:
|
| 47 |
-
test_range:
|
| 48 |
copy_files: false # copy instead of move
|
| 49 |
|
| 50 |
# Optional — only needed if you're training, not for running inference
|
| 51 |
# against a released checkpoint. Requires steps.split to have run (computes
|
| 52 |
# stats from the train split only).
|
| 53 |
sxr_normalization:
|
| 54 |
-
compute:
|
| 55 |
-
output_path: "/
|
|
|
|
| 10 |
wavelengths: [94, 131, 171, 193, 211, 304, 335]
|
| 11 |
|
| 12 |
aia:
|
| 13 |
+
raw_dir: "./foxes_data/AIA_raw" # raw FITS from download/download_sdo.py
|
| 14 |
+
bad_files_dir: "./foxes_data/AIA_bad" # bad-timestamp FITS get moved here
|
| 15 |
+
processed_dir: "./foxes_data/AIA_processed" # paired 512x512 .npy stacks
|
| 16 |
|
| 17 |
sxr:
|
| 18 |
+
raw_dir: "./foxes_data/SXR_raw" # raw netCDF from download/download_sxr.py
|
| 19 |
+
combined_dir: "./foxes_data/SXR_combined" # one interpolated CSV per satellite
|
| 20 |
|
| 21 |
output:
|
| 22 |
+
sxr_dir: "./foxes_data/SXR_processed" # per-timestamp xrsb_flux .npy, aligned to AIA
|
| 23 |
+
aia_missing_dir: "./foxes_data/AIA_missing" # AIA files with no matching SXR value
|
| 24 |
|
| 25 |
processing:
|
| 26 |
max_processes: null # cap worker processes; null = use all CPUs
|
|
|
|
| 34 |
convert_aia: true # raw AIA FITS -> paired .npy stacks
|
| 35 |
combine_sxr: true # combine raw multi-satellite GOES files into per-satellite CSVs
|
| 36 |
align: true # match AIA timestamps to GOES CSVs -> per-timestamp SXR .npy
|
| 37 |
+
split: false # split aia.processed_dir / output.sxr_dir into train/val/test
|
| 38 |
|
| 39 |
# Only needed if you're training (training/train_config.yaml expects
|
| 40 |
# aia_dir/sxr_dir to each have train/val/test subfolders) — not needed for
|
|
|
|
| 42 |
# aia.processed_dir and output.sxr_dir in place, using the same date ranges
|
| 43 |
# so a given timestamp lands in the same split for both.
|
| 44 |
split:
|
| 45 |
+
train_range: null # e.g. ["2012-01-01", "2022-12-31"]; null with the others -> month-based default
|
| 46 |
+
val_range: null # e.g. ["2023-01-01", "2023-06-30"]
|
| 47 |
+
test_range: null # e.g. ["2023-08-01", "2023-08-31"]
|
| 48 |
copy_files: false # copy instead of move
|
| 49 |
|
| 50 |
# Optional — only needed if you're training, not for running inference
|
| 51 |
# against a released checkpoint. Requires steps.split to have run (computes
|
| 52 |
# stats from the train split only).
|
| 53 |
sxr_normalization:
|
| 54 |
+
compute: false
|
| 55 |
+
output_path: "./foxes_data/normalized_sxr.npy"
|
download/hf_download_config.yaml
CHANGED
|
@@ -18,8 +18,8 @@ repo_id: "griffingoodwin04/FOXES"
|
|
| 18 |
# {aia_dir}/{local_split}/{filename} ("validation" maps to local dir "val")
|
| 19 |
# {sxr_dir}/{local_split}/{filename}
|
| 20 |
# -----------------------------------------------------------------------------
|
| 21 |
-
aia_dir: "/
|
| 22 |
-
sxr_dir: "/
|
| 23 |
|
| 24 |
# -----------------------------------------------------------------------------
|
| 25 |
# Splits to download (any subset of: train, validation, test)
|
|
|
|
| 18 |
# {aia_dir}/{local_split}/{filename} ("validation" maps to local dir "val")
|
| 19 |
# {sxr_dir}/{local_split}/{filename}
|
| 20 |
# -----------------------------------------------------------------------------
|
| 21 |
+
aia_dir: "./foxes_data/AIA_hg_processed"
|
| 22 |
+
sxr_dir: "./foxes_data/SXR_hg_processed"
|
| 23 |
|
| 24 |
# -----------------------------------------------------------------------------
|
| 25 |
# Splits to download (any subset of: train, validation, test)
|
download/sdo_download_config.yaml
CHANGED
|
@@ -10,11 +10,11 @@
|
|
| 10 |
# python download/download_sdo.py --config download/sdo_download_config.yaml
|
| 11 |
# =============================================================================
|
| 12 |
|
| 13 |
-
download_dir: "/
|
| 14 |
email: "ggoodwin5@gsu.edu" # must be registered with JSOC: http://jsoc.stanford.edu/ajax/register_email.html
|
| 15 |
|
| 16 |
start_date: "2023-08-01 00:00:00"
|
| 17 |
end_date: "2023-12-01 00:00:00"
|
| 18 |
-
cadence:
|
| 19 |
|
| 20 |
wavelengths: [94, 131, 171, 193, 211, 304, 335]
|
|
|
|
| 10 |
# python download/download_sdo.py --config download/sdo_download_config.yaml
|
| 11 |
# =============================================================================
|
| 12 |
|
| 13 |
+
download_dir: "./foxes_data/AIA_raw"
|
| 14 |
email: "ggoodwin5@gsu.edu" # must be registered with JSOC: http://jsoc.stanford.edu/ajax/register_email.html
|
| 15 |
|
| 16 |
start_date: "2023-08-01 00:00:00"
|
| 17 |
end_date: "2023-12-01 00:00:00"
|
| 18 |
+
cadence: 60 # minutes between samples
|
| 19 |
|
| 20 |
wavelengths: [94, 131, 171, 193, 211, 304, 335]
|
download/sxr_download_config.yaml
CHANGED
|
@@ -11,7 +11,7 @@
|
|
| 11 |
# python download/download_sxr.py --config download/sxr_download_config.yaml
|
| 12 |
# =============================================================================
|
| 13 |
|
| 14 |
-
save_dir: "/
|
| 15 |
|
| 16 |
start_date: "2023-08-01"
|
| 17 |
end_date: "2023-08-31"
|
|
|
|
| 11 |
# python download/download_sxr.py --config download/sxr_download_config.yaml
|
| 12 |
# =============================================================================
|
| 13 |
|
| 14 |
+
save_dir: "./foxes_data/SXR_raw"
|
| 15 |
|
| 16 |
start_date: "2023-08-01"
|
| 17 |
end_date: "2023-08-31"
|
training/train_config.yaml
CHANGED
|
@@ -91,10 +91,10 @@ callbacks:
|
|
| 91 |
attention_log_every_n_epochs: 1 # how often (in epochs) to log attention maps
|
| 92 |
|
| 93 |
data:
|
| 94 |
-
aia_dir: "/
|
| 95 |
-
sxr_dir: "/
|
| 96 |
-
sxr_norm_path: "/
|
| 97 |
-
checkpoints_dir: "/
|
| 98 |
num_workers: 8 # DataLoader worker processes; defaults to min(8, cpu_count()) if omitted
|
| 99 |
|
| 100 |
wandb:
|
|
|
|
| 91 |
attention_log_every_n_epochs: 1 # how often (in epochs) to log attention maps
|
| 92 |
|
| 93 |
data:
|
| 94 |
+
aia_dir: "./foxes_data/AIA_processed"
|
| 95 |
+
sxr_dir: "./foxes_data/SXR_processed"
|
| 96 |
+
sxr_norm_path: "./foxes_data/normalized_sxr.npy"
|
| 97 |
+
checkpoints_dir: "./foxes_data/chkpt/" # update to actual checkpoint
|
| 98 |
num_workers: 8 # DataLoader worker processes; defaults to min(8, cpu_count()) if omitted
|
| 99 |
|
| 100 |
wandb:
|